commit: aba0962b8f901301302a2fefacac14f8cd34a51b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 27 19:32:09 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Jul 27 19:32:09 2022 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aba0962b
Move UpdateRepos to repos.py and support gitlab stage
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 4 +-
buildbot_gentoo_ci/db/model.py | 12 ++++-
buildbot_gentoo_ci/db/repositorys.py | 19 +++++++-
buildbot_gentoo_ci/steps/builders.py | 40 ----------------
buildbot_gentoo_ci/steps/repos.py | 73 +++++++++++++++++++++++++++++-
5 files changed, 102 insertions(+), 46 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 67447aa..7f2dff9 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -46,7 +46,7 @@ def update_db_cpv():
# set needed Propertys
f.addStep(package.SetupPropertys())
# update the repositorys listed in project_repository
- f.addStep(builders.UpdateRepos())
+ f.addStep(repos.UpdateRepos())
# add repo.conf
#f.addStep(portage.SetReposConf())
# if categorys in db
@@ -99,7 +99,7 @@ def run_build_request():
# set needed Propertys
f.addStep(builders.SetupPropertys())
# update the repositorys listed in project_repository
- f.addStep(builders.UpdateRepos())
+ f.addStep(repos.UpdateRepos())
# Clean and add new /etc/portage
#NOTE: remove the symlink befor the dir
#f.addStep(buildbot_steps.ShellCommand(
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 908ebb5..d6e9860 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -71,10 +71,18 @@ class Model(base.DBConnectorComponent):
# description of the repository
sa.Column('description', sa.Text, nullable=True),
sa.Column('url', sa.String(255), nullable=True),
- sa.Column('type', sa.Enum('gitpuller'), nullable=False,
default='gitpuller'),
+ sa.Column('branch', sa.String(255), nullable=True),
+ sa.Column('type', sa.Enum('git', 'gitlab'), nullable=False,
default='git'),
+ sa.Column('mode', sa.Enum('full', 'incremental'), nullable=False,
default='incremental'),
+ sa.Column('method', sa.Enum('clobber', 'fresh', 'clean', 'copy'),
nullable=False, default='fresh'),
+ sa.Column('alwaysuselatest', sa.Boolean, default=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
- sa.Column('ebuild', sa.Boolean, default=False),
+ sa.Column('ebuild', sa.Boolean, default=True),
+ sa.Column('merge', sa.Boolean, default=False),
+ # filenames for SecretInAFile
+ sa.Column('sshprivatekey', sa.String(50), nullable=True),
+ sa.Column('sshhostkey', sa.String(50), nullable=True),
)
# Use by GitPoller
diff --git a/buildbot_gentoo_ci/db/repositorys.py
b/buildbot_gentoo_ci/db/repositorys.py
index 8bac1a1..4d5d048 100644
--- a/buildbot_gentoo_ci/db/repositorys.py
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -91,15 +91,32 @@ class
RepositorysConnectorComponent(base.DBConnectorComponent):
yield self.db.pool.do(thd)
def _row2dict(self, conn, row):
+ if row.branch == 'none':
+ branch = False
+ else:
+ branch = row.branch
+ if row.sshprivatekey == 'none':
+ sshprivatekey = False
+ sshhostkey = False
+ else:
+ sshprivatekey = row.sshprivatekey
+ sshhostkey = row.sshhostkey
return dict(
uuid=row.uuid,
name=row.name,
description=row.description,
url=row.url,
+ type=row.type,
+ branch=branch,
+ mode=row.mode,
+ method=row.method,
+ alwaysuselatest=row.alwaysuselatest,
auto=row.auto,
enabled=row.enabled,
ebuild=row.ebuild,
- type=row.type
+ merge=row.merge,
+ sshprivatekey=sshprivatekey,
+ sshhostkey=sshhostkey
)
def _row2dict_gitpuller(self, conn, row):
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 383ae60..a54c68c 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -343,46 +343,6 @@ class SetupPropertys(BuildStep):
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for
project', self.getProperty('project_data')['name']])
return SUCCESS
-# Should be moved to repos.py
-class UpdateRepos(BuildStep):
-
- name = 'UpdateRepos'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
-
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
- @defer.inlineCallbacks
- def run(self):
- self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
- portage_repos_path = self.getProperty('portage_repos_path')
- project_data = self.getProperty('project_data')
- # update/add all repos that in project_repository for the project
- projects_repositorys_data = yield
self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
- for project_repository_data in projects_repositorys_data:
- repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
- if self.getProperty('rootworkdir'):
- repository_path =
os.path.join(self.getProperty('rootworkdir'), portage_repos_path[1:],
repository_data['name'])
- else:
- repository_path = os.path.join(portage_repos_path,
repository_data['name'], '')
- yield self.build.addStepsAfterCurrentStep([
- steps.Git(repourl=repository_data['url'],
- name = 'Git pull ' + repository_data['name'],
- mode='full',
- submodules=True,
- alwaysUseLatest=True,
- workdir=repository_path,
- #FIXME: set filenames in repositorys db
- sshPrivateKey =
util.Secret("gitlab.gentoo.org_gentoo-ci.priv"),
- sshHostKey = util.Secret("gitlab.gentoo.org.host")
- )
- ])
- return SUCCESS
-
class RunEmerge(BuildStep):
description = 'Running'
diff --git a/buildbot_gentoo_ci/steps/repos.py
b/buildbot_gentoo_ci/steps/repos.py
index ae63c78..5b4b623 100644
--- a/buildbot_gentoo_ci/steps/repos.py
+++ b/buildbot_gentoo_ci/steps/repos.py
@@ -11,7 +11,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
-from buildbot.plugins import steps
+from buildbot.plugins import steps, util
from buildbot.config import error as config_error
class CheckPathRepositoryLocal(BuildStep):
@@ -168,3 +168,74 @@ class CheckRepository(BuildStep):
return FAILURE
#yield
self.gentooci.db.repositorys.updateGitPollerTime(repository_uuid)
return SUCCESS
+
+class UpdateRepos(BuildStep):
+
+ name = 'UpdateRepos'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ portage_repos_path = self.getProperty('portage_repos_path')
+ project_data = self.getProperty('project_data')
+ # update/add all repos that in project_repository for the project
+ projects_repositorys_data = yield
self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
+ for project_repository_data in projects_repositorys_data:
+ repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
+ print(repository_data)
+ if repository_data['auto'] and repository_data['enabled']:
+ if self.getProperty('rootworkdir'):
+ repository_path =
os.path.join(self.getProperty('rootworkdir'), portage_repos_path[1:],
repository_data['name'])
+ else:
+ repository_path = os.path.join(portage_repos_path,
repository_data['name'], '')
+ if repository_data['branch']:
+ branch = repository_data['branch']
+ else:
+ branch = 'HEAD'
+ # filenames to use with Secret
+ if repository_data['sshprivatekey']:
+ sshprivatekey =
util.Secret(repository_data['sshprivatekey'])
+ sshhostkey = util.Secret(repository_data['sshhostkey'])
+ else:
+ sshprivatekey = None
+ sshhostkey = None
+ GitdescriptionDone = ' '.join([repository_data['type'],
'pull', repository_data['name'], branch])
+ if repository_data['type'] == 'git':
+ yield self.build.addStepsAfterCurrentStep([
+ steps.Git(repourl=repository_data['url'],
+ name = 'RunGit',
+ descriptionDone=GitdescriptionDone,
+ branch = branch,
+ mode=repository_data['mode'],
+ method=repository_data['method'],
+ submodules=True,
+ alwaysUseLatest=repository_data['alwaysuselatest'],
+ workdir=repository_path,
+ sshPrivateKey = sshprivatekey,
+ sshHostKey = sshhostkey
+ )
+ ])
+ if repository_data['type'] =='gitlab':
+ yield self.build.addStepsAfterCurrentStep([
+ steps.GitLab(repourl=repository_data['url'],
+ name = 'RunGit',
+ descriptionDone=GitdescriptionDone,
+ branch = branch,
+ mode=repository_data['mode'],
+ method=repository_data['method'],
+ submodules=True,
+ alwaysUseLatest=repository_data['alwaysuselatest'],
+ workdir=repository_path,
+ sshPrivateKey = sshprivatekey,
+ sshHostKey = sshhostkey
+ )
+ ])
+ return SUCCESS
