[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
commit: aba0962b8f901301302a2fefacac14f8cd34a51b
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 27 19:32:09 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 27 19:32:09 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aba0962b
Move UpdateRepos to repos.py and support gitlab stage
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 4 +-
buildbot_gentoo_ci/db/model.py | 12 -
buildbot_gentoo_ci/db/repositorys.py | 19 +++-
buildbot_gentoo_ci/steps/builders.py | 40
buildbot_gentoo_ci/steps/repos.py | 73 +-
5 files changed, 102 insertions(+), 46 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 67447aa..7f2dff9 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -46,7 +46,7 @@ def update_db_cpv():
# set needed Propertys
f.addStep(package.SetupPropertys())
# update the repositorys listed in project_repository
-f.addStep(builders.UpdateRepos())
+f.addStep(repos.UpdateRepos())
# add repo.conf
#f.addStep(portage.SetReposConf())
# if categorys in db
@@ -99,7 +99,7 @@ def run_build_request():
# set needed Propertys
f.addStep(builders.SetupPropertys())
# update the repositorys listed in project_repository
-f.addStep(builders.UpdateRepos())
+f.addStep(repos.UpdateRepos())
# Clean and add new /etc/portage
#NOTE: remove the symlink befor the dir
#f.addStep(buildbot_steps.ShellCommand(
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 908ebb5..d6e9860 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -71,10 +71,18 @@ class Model(base.DBConnectorComponent):
# description of the repository
sa.Column('description', sa.Text, nullable=True),
sa.Column('url', sa.String(255), nullable=True),
-sa.Column('type', sa.Enum('gitpuller'), nullable=False,
default='gitpuller'),
+sa.Column('branch', sa.String(255), nullable=True),
+sa.Column('type', sa.Enum('git', 'gitlab'), nullable=False,
default='git'),
+sa.Column('mode', sa.Enum('full', 'incremental'), nullable=False,
default='incremental'),
+sa.Column('method', sa.Enum('clobber', 'fresh', 'clean', 'copy'),
nullable=False, default='fresh'),
+sa.Column('alwaysuselatest', sa.Boolean, default=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
-sa.Column('ebuild', sa.Boolean, default=False),
+sa.Column('ebuild', sa.Boolean, default=True),
+sa.Column('merge', sa.Boolean, default=False),
+# filenames for SecretInAFile
+sa.Column('sshprivatekey', sa.String(50), nullable=True),
+sa.Column('sshhostkey', sa.String(50), nullable=True),
)
# Use by GitPoller
diff --git a/buildbot_gentoo_ci/db/repositorys.py
b/buildbot_gentoo_ci/db/repositorys.py
index 8bac1a1..4d5d048 100644
--- a/buildbot_gentoo_ci/db/repositorys.py
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -91,15 +91,32 @@ class
RepositorysConnectorComponent(base.DBConnectorComponent):
yield self.db.pool.do(thd)
def _row2dict(self, conn, row):
+if row.branch == 'none':
+branch = False
+else:
+branch = row.branch
+if row.sshprivatekey == 'none':
+sshprivatekey = False
+sshhostkey = False
+else:
+sshprivatekey = row.sshprivatekey
+sshhostkey = row.sshhostkey
return dict(
uuid=row.uuid,
name=row.name,
description=row.description,
url=row.url,
+type=row.type,
+branch=branch,
+mode=row.mode,
+method=row.method,
+alwaysuselatest=row.alwaysuselatest,
auto=row.auto,
enabled=row.enabled,
ebuild=row.ebuild,
-type=row.type
+merge=row.merge,
+sshprivatekey=sshprivatekey,
+sshhostkey=sshhostkey
)
def _row2dict_gitpuller(self, conn, row):
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 383ae60..a54c68c 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -343,46 +343,6 @@ class SetupPropertys(BuildStep):
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for
project', self.getProperty('project_data')['name']])
return SUCCESS
-# Should be moved to repos.py
-class UpdateRepos(BuildStep):
-
-name = 'UpdateRepos'
-description = 'Running'
-descriptionDone = 'Ran'
-descriptionSuffix = None
-
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
commit: 5176761ad56557e45359729d7b2989dadeace7b5
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Mar 11 01:14:02 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Mar 11 01:14:02 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5176761a
Add GitPuller to change_source and change cpv_changes to git_changes
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/change_source.py | 18 ++
buildbot_gentoo_ci/config/schedulers.py| 40 +
buildbot_gentoo_ci/db/model.py | 10 +++-
buildbot_gentoo_ci/db/repositorys.py | 48 ++-
buildbot_gentoo_ci/steps/update_db.py | 95 +++---
5 files changed, 134 insertions(+), 77 deletions(-)
diff --git a/buildbot_gentoo_ci/config/change_source.py
b/buildbot_gentoo_ci/config/change_source.py
new file mode 100644
index 000..454ed4a
--- /dev/null
+++ b/buildbot_gentoo_ci/config/change_source.py
@@ -0,0 +1,18 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from buildbot.plugins import changes, util
+
+#FIXME:
+# Get the repositorys info from the gentoo-ci db
+def gentoo_change_source(cs=[]):
+cs.append(changes.GitPoller(
+repourl='https://github.com/gentoo/gentoo.git',
+branches=True,
+workdir= 'repositorys' + '/gentoo.git/',
+pollInterval=240,
+pollRandomDelayMin=20,
+pollRandomDelayMax=60,
+project='gentoo'
+))
+return cs
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index 7cfb122..0cbec96 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -4,7 +4,7 @@
from buildbot.plugins import schedulers, util
@util.renderer
-def builderUpdateDbNames(self, props):
+def builderUpdateDbNames(props):
builders = set()
for f in props.files:
if f.endswith('.ebuild'):
@@ -12,22 +12,38 @@ def builderUpdateDbNames(self, props):
return list(builders)
@util.renderer
-def cpvUpdateDb(props):
-cpv_changes = []
-for f in props.files:
-if f.endswith('.ebuild'):
-cppv = f.split('.eb', 0)
-cpv = cppv.split('/', 0) + '/' + cppv.split('/', 2)
-if not cpv in cpv_changes:
-cpv_changes.append(cpv)
-return cpv_changes
+def gitUpdateDb(props):
+git_changes = []
+print(props.changes)
+for k in props.changes:
+change_data = {}
+print(k)
+change_data['cpvs'] = []
+for v in k['files']:
+if v.endswith('.ebuild'):
+c = v.split('/')[0]
+pv = v.split('/')[2][:-7]
+cpv = c + '/' + pv
+print(cpv)
+change_data['cpvs'].append(cpv)
+if k['repository'].endswith('.git'):
+for v in k['repository'].split('/'):
+if v.endswith('.git'):
+change_data['repository'] = v[:-4]
+change_data['author'] = k['author']
+change_data['committer'] = k['committer']
+change_data['comments'] = k['comments']
+change_data['revision'] = k['revision']
+git_changes.append(change_data)
+print(git_changes)
+return git_changes
def gentoo_schedulers():
scheduler_update_db = schedulers.SingleBranchScheduler(
name='scheduler_update_db',
treeStableTimer=60,
properties = {
-'cpv_changes' : cpvUpdateDb,
+'git_changes' : gitUpdateDb,
},
builderNames = builderUpdateDbNames,
change_filter=util.ChangeFilter(branch='master'),
@@ -60,7 +76,7 @@ def gentoo_schedulers():
builderNames=["run_build_request"])
s = []
s.append(test_updatedb)
-#s.append(scheduler_update_db)
+s.append(scheduler_update_db)
s.append(update_cpv_data)
s.append(update_v_data)
s.append(build_request_data)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 2221ba4..939059c 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -71,14 +71,15 @@ class Model(base.DBConnectorComponent):
# description of the repository
sa.Column('description', sa.Text, nullable=True),
sa.Column('mirror_url', sa.String(255), nullable=True),
+sa.Column('type', sa.Enum('gitpuller'), nullable=False,
default='gitpuller'),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
sa.Column('ebuild', sa.Boolean, default=False),
)
# Use by GitPoller
-repository_gitpuller = sautils.Table(
-"repository_gitpuller", metadata,
+repositorys_gitpullers = sautils.Table(
+"repositorys_gitpullers",
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
commit: 22ad0178cca549026ab51f9f26e3fb9ad28f6374
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Feb 27 22:45:35 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Feb 27 22:45:35 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=22ad0178
Add SetPackageDefault and SetEnvDefault to steps
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 6 +-
buildbot_gentoo_ci/db/model.py | 27 -
buildbot_gentoo_ci/db/portages.py | 14 +
buildbot_gentoo_ci/db/projects.py | 44 ++
buildbot_gentoo_ci/steps/portage.py| 97 ++
5 files changed, 185 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index ac8bac1..acafed8 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -9,6 +9,7 @@ from buildbot_gentoo_ci.steps import category
from buildbot_gentoo_ci.steps import package
from buildbot_gentoo_ci.steps import version
from buildbot_gentoo_ci.steps import builders
+from buildbot_gentoo_ci.steps import portage
def update_db_check():
f = util.BuildFactory()
@@ -114,9 +115,10 @@ def run_build_request():
f.addStep(builders.UpdateRepos())
# setup make.conf
f.addStep(builders.SetMakeConf())
-# setup package.*
-#f.addStep(portages.SetPackageUse())
# setup env
+f.addStep(portage.SetEnvDefault())
+# setup package.*
+#f.addStep(portage.SetPackageDefault())
# setup files in /etc if needed
# run --regen if needed on repo
# update packages before any tests
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 2153b79..b91f158 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -139,7 +139,6 @@ class Model(base.DBConnectorComponent):
sa.Column('project_uuid', sa.String(36),
sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
nullable=False),
-# FIXME: directorys should be moved to own table
sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'),
nullable=False),
sa.Column('value', sa.String(255), nullable=False),
)
@@ -162,6 +161,32 @@ class Model(base.DBConnectorComponent):
sa.Column('value', sa.String(255), nullable=False),
)
+# projects etc/portage/env settings
+projects_portages_env = sautils.Table(
+"projects_portages_env", metadata,
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+sa.Column('makeconf_id', sa.String(255),
+ sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'),
+ nullable=False),
+sa.Column('name', sa.String(255), nullable=False),
+sa.Column('value', sa.String(255), nullable=False),
+)
+
+# projects etc/portage/package.* settings
+projects_portage_package = sautils.Table(
+"projects_portage_package", metadata,
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+sa.Column('directorys', sa.Enum('use', 'accept_keywords', 'env'),
nullable=False),
+sa.Column('value1', sa.String(255), nullable=False),
+sa.Column('value2', sa.String(255), nullable=True),
+)
+
projects_emerge_options = sautils.Table(
"projects_emerge_options", metadata,
sa.Column('id', sa.Integer, primary_key=True),
diff --git a/buildbot_gentoo_ci/db/portages.py
b/buildbot_gentoo_ci/db/portages.py
index 428fb84..86321e3 100644
--- a/buildbot_gentoo_ci/db/portages.py
+++ b/buildbot_gentoo_ci/db/portages.py
@@ -36,6 +36,20 @@ class PortagesConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+@defer.inlineCallbacks
+def getVariableById(self, id):
+def thd(conn):
+tbl = self.db.model.portages_makeconf
+q = tbl.select()
+q = q.where(tbl.c.id == id)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
def _row2dict(self, conn, row):
return dict(
id=row.id,
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 37d401c..37ae2f1 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -130,6 +130,28 @@ class
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, /, buildbot_gentoo_ci/config/
commit: d9dcf85c92b58ca427123ef2c38b8374ecdae293
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Jan 10 15:24:41 2021 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Jan 10 15:24:41 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d9dcf85c
Add BuildFactory update_db_check
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/builders.py | 18 +++-
buildbot_gentoo_ci/config/buildfactorys.py | 66
buildbot_gentoo_ci/config/config.py| 2 +-
buildbot_gentoo_ci/config/schedulers.py| 9 +-
buildbot_gentoo_ci/config/service.py | 6 +-
buildbot_gentoo_ci/db/connector.py | 7 +-
buildbot_gentoo_ci/db/model.py | 6 +-
buildbot_gentoo_ci/db/projects.py | 69
buildbot_gentoo_ci/db/repositorys.py | 73 +
buildbot_gentoo_ci/steps/__init__.py | 0
buildbot_gentoo_ci/steps/update_db.py | 162 +
gentooci.cfg | 2 +-
12 files changed, 383 insertions(+), 37 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index 114bb83..c069841 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Gentoo Authors
+# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from buildbot.plugins import util
@@ -7,9 +7,19 @@ from buildbot_gentoo_ci.config import buildfactorys
def gentoo_builders(b=[]):
# FIXME: get workers from db
b.append(util.BuilderConfig(
-name='update_db_packages',
-workername=[ 'updatedb_1', ],
-factory=buildfactorys.f_update_db_packages()
+name='update_db_check',
+workername='updatedb_1',
+factory=buildfactorys.update_db_check()
+)
+)
+# FIXME: get workers from db
+# Use multiplay workers depend on Property(cpv)
+# if cp do not match next one, use diffrent worker then
+# or first cp have done its buildsteps.
+b.append(util.BuilderConfig(
+name='update_cpv_data',
+workername='updatedb_1',
+factory=buildfactorys.update_db_cp()
)
)
return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index ef479a2..3fae376 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -1,28 +1,50 @@
-# Copyright 2020 Gentoo Authors
+# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-from buildbot.plugins import util, steps
-
-def f_update_db_packages():
+from buildbot.plugins import steps as buildbot_steps
+from buildbot.plugins import util
+
+from buildbot_gentoo_ci.steps import update_db
+#from buildbot_gentoo_ci.steps import category
+#from buildbot_gentoo_ci.steps import package
+
+def update_db_check():
f = util.BuildFactory()
-# FIXME:
-# Get base project from db
-# Check if base project repo for is cloned
-# Check if gentoo repo is cloned
+# FIXME: 1
+# Get base project data from db
+# return project_repository, profile_repository,
+# project
+f.addStep(update_db.GetDataGentooCiProject())
+# Check if base project repo is cloned
+# Check if profile repo is cloned
# check if the profile link is there
+# check if the repository is cloned
+f.addStep(update_db.CheckPathGentooCiProject())
# check if etc/portage has no error
-# Get the repository some way
-repository=util.Property('repository')
-# For loop
-# for cpv in util.Property('cpv_changes')
-# check if not categorys in db
-# run update categorys db step
-# check if not package in db
-# run update package db step
-# and get foo{cpv] with metadata back
-# if new or updated package
-# else
-# run update package db step in a pool of workers
-# and get foo{cpv] with metadata back
-# if new or updated package
+# return config_root
+f.addStep(update_db.CheckProjectGentooCiProject())
+# Make a for loop and trigger new builders with cpv from cpv_changes
+# return cpv, repository, project_data, config_root
+f.addStep(update_db.CheckCPVGentooCiProject())
+return f
+
+def update_db_cp():
+f = util.BuildFactory()
+# FIXME: 2
+# if categorys in db
+# return category_data
+# add check category path step at end
+# else
+# add category to db step
+# return category_data
+#f.addStep(category.CheckCGentooCiProject())
+# if package in db
+# return package_data
+# add check package path step at end
+# else
+# add package to db step
+# return package_data
+
