commit: a4a5adaa76b48ae04c142f0c1bf602c210892731
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 11 17:45:46 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jan 11 17:45:46 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=a4a5adaa
Add CheckPGentooCiProject on BuildFactory update_db_cp
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 5 +-
buildbot_gentoo_ci/db/connector.py | 2 +
buildbot_gentoo_ci/db/packages.py | 80 ++++++++++++++++++++++++++++++
buildbot_gentoo_ci/steps/category.py | 10 ++--
buildbot_gentoo_ci/steps/package.py | 65 ++++++++++++++++++++++++
buildbot_gentoo_ci/steps/update_db.py | 30 +++++------
6 files changed, 170 insertions(+), 22 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 4a0ffcd..4038bb1 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -6,7 +6,7 @@ from buildbot.plugins import util
from buildbot_gentoo_ci.steps import update_db
from buildbot_gentoo_ci.steps import category
-#from buildbot_gentoo_ci.steps import package
+from buildbot_gentoo_ci.steps import package
def update_db_check():
f = util.BuildFactory()
@@ -44,7 +44,8 @@ def update_db_cp():
# else
# add package to db step
# return package_data
- #f.addStep(package.CheckPGentooCiProject())
+ f.addStep(package.CheckPGentooCiProject())
# Make a for loop and trigger new builders with v from cpv
# return package_data, cpv, repository, project_data, config_root
+ #f.addStep(package.TriggerVGentooCiProject())
return f
diff --git a/buildbot_gentoo_ci/db/connector.py
b/buildbot_gentoo_ci/db/connector.py
index c10fc49..0121d11 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -33,6 +33,7 @@ from buildbot_gentoo_ci.db import model
from buildbot_gentoo_ci.db import projects
from buildbot_gentoo_ci.db import repositorys
from buildbot_gentoo_ci.db import categorys
+from buildbot_gentoo_ci.db import packages
upgrade_message = textwrap.dedent("""\
@@ -75,6 +76,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.projects = projects.ProjectsConnectorComponent(self)
self.repositorys = repositorys.RepositorysConnectorComponent(self)
self.categorys = categorys.CategorysConnectorComponent(self)
+ self.packages = packages.PackagesConnectorComponent(self)
@defer.inlineCallbacks
def setup(self, config, check_version=True, verbose=True):
diff --git a/buildbot_gentoo_ci/db/packages.py
b/buildbot_gentoo_ci/db/packages.py
new file mode 100644
index 0000000..13cc267
--- /dev/null
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -0,0 +1,80 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class PackagesConnectorComponent(base.DBConnectorComponent):
+
+ @defer.inlineCallbacks
+ def getPackageByName(self, name):
+ def thd(conn):
+ tbl = self.db.model.packages
+ q = tbl.select()
+ q = q.where(tbl.c.name == name)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getPackageByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.packages
+ q = tbl.select()
+ q = q.where(tbl.c.uuid == uuid)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def addPackage(self, name, repository_uuid, category_uuid):
+ def thd(conn, no_recurse=False):
+ try:
+ tbl = self.db.model.packages
+ q = tbl.insert()
+ r = conn.execute(q, dict(name=name,
+ repository_uuid=repository_uuid,
+ category_uuid=category_uuid))
+ except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+ uuid = None
+ else:
+ uuid = r.inserted_primary_key[0]
+ return uuid
+ res = yield self.db.pool.do(thd)
+ return res
+
+ def _row2dict(self, conn, row):
+ return dict(
+ uuid=row.uuid,
+ name=row.name,
+ repository_uuid=row.repository_uuid,
+ category_uuid=row.category_uuid
+ )
diff --git a/buildbot_gentoo_ci/steps/category.py
b/buildbot_gentoo_ci/steps/category.py
index e641a97..1aca99a 100644
--- a/buildbot_gentoo_ci/steps/category.py
+++ b/buildbot_gentoo_ci/steps/category.py
@@ -21,15 +21,14 @@ class AddCategory(BuildStep):
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
- self.category = self.getProperty("category")
self.category_data = {}
- self.category_data['name'] = self.category
- self.category_data['uuid'] = yield
self.gentooci.db.categorys.addCategory(self.category)
+ self.category_data['name'] = self.getProperty("category")
+ self.category_data['uuid'] = yield
self.gentooci.db.categorys.addCategory(self.category_data['name'])
print(self.category_data)
self.setProperty("category_data", self.category_data, 'category_data')
self.setProperty("config_root", self.getProperty("config_root"),
'config_root')
self.setProperty("project_data", self.getProperty("project_data"),
'project_data')
- self.setProperty("repository", self.getProperty("repository"),
'repository')
+ self.setProperty("repository_data",
self.getProperty("repository_data"), 'repository_data')
self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
return SUCCESS
@@ -42,10 +41,11 @@ class CheckCGentooCiProject(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.category = yield self.getProperty("cpv").split('/')[0]
print(self.category)
+ print(self.getProperty("repository"))
self.category_data = yield
self.gentooci.db.categorys.getCategoryByName(self.category)
self.setProperty("config_root", self.getProperty("config_root"),
'config_root')
self.setProperty("project_data", self.getProperty("project_data"),
'project_data')
- self.setProperty("repository", self.getProperty("repository"),
'repository')
+ self.setProperty("repository_data",
self.getProperty("repository_data"), 'repository_data')
self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
print(self.category_data)
if self.category_data is None:
diff --git a/buildbot_gentoo_ci/steps/package.py
b/buildbot_gentoo_ci/steps/package.py
new file mode 100644
index 0000000..916bb73
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -0,0 +1,65 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import re
+import os
+
+from portage.xml.metadata import MetaDataXML
+from portage.checksum import perform_checksum
+from portage.versions import catpkgsplit
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+
+class AddPackage(BuildStep):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ self.package_data = {}
+ self.package_data['name'] = self.getProperty("package")
+ self.package_data['repository_uuid'] =
self.getProperty("repository_data")['uuid']
+ self.package_data['category_uuid'] =
self.getProperty("category_data")['uuid']
+ self.package_data['uuid'] = yield self.gentooci.db.packages.addPackage(
+ self.package_data['name'],
+
self.package_data['repository_uuid'],
+ self.package_data['category_uuid']
+ )
+ print(self.package_data)
+ self.setProperty("package_data", self.package_data, 'package_data')
+ self.setProperty("config_root", self.getProperty("config_root"),
'config_root')
+ self.setProperty("project_data", self.getProperty("project_data"),
'project_data')
+ self.setProperty("repository_data",
self.getProperty("repository_data"), 'repository_data')
+ self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
+ return SUCCESS
+
+class CheckPGentooCiProject(BuildStep):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ self.package = yield catpkgsplit(self.getProperty("cpv"))[1]
+ print(self.package)
+ self.package_data = yield
self.gentooci.db.packages.getPackageByName(self.package)
+ self.setProperty("config_root", self.getProperty("config_root"),
'config_root')
+ self.setProperty("project_data", self.getProperty("project_data"),
'project_data')
+ self.setProperty("repository_data",
self.getProperty("repository_data"), 'repository_data')
+ self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
+ print(self.package_data)
+ if self.package_data is None:
+ self.setProperty("category_data",
self.getProperty("category_data"), 'category_data')
+ self.setProperty("package", self.package, 'package')
+ yield self.build.addStepsAfterCurrentStep([AddPackage()])
+ #yield self.build.addStepsAfterLastStep([AddMetadataPackage()])
+ return SUCCESS
+ self.setProperty("package_data", self.package_data, 'package_data')
+ #yield self.build.addStepsAfterLastStep([CheckPathPackage()])
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/update_db.py
b/buildbot_gentoo_ci/steps/update_db.py
index df28f35..d00407c 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -1,5 +1,6 @@
# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+
import os
from portage import config as portage_config
@@ -36,16 +37,19 @@ class GetDataGentooCiProject(BuildStep):
if self.profile_repository_data is None:
log.err('No data for repository in the database')
return FAILURE
+ #self.repository = self.getProperty("repository")
+ self.repository = 'gentoo'
+ self.repository_data = yield
self.gentooci.db.repositorys.getRepositoryByName(self.repository)
print(self.project_data)
print(self.project_repository_data)
print(self.profile_repository_data)
print(self.getProperty("cpv_changes"))
- print(self.getProperty("repository"))
+ print(self.repository_data)
self.setProperty("project_data", self.project_data, 'project_data')
self.setProperty("project_repository_data",
self.project_repository_data, 'project_repository_data')
self.setProperty("profile_repository_data",
self.profile_repository_data, 'profile_repository_data')
self.setProperty("cpv_changes", self.getProperty("cpv_changes"),
'cpv_changes')
- self.setProperty("repository", self.getProperty("repository"),
'repository')
+ self.setProperty("repository_data", self.repository_data,
'repository_data')
return SUCCESS
class CheckPathGentooCiProject(BuildStep):
@@ -59,18 +63,17 @@ class CheckPathGentooCiProject(BuildStep):
self.repository_basedir =
self.gentooci.config.project['repository_basedir']
self.profile_repository_data =
self.getProperty("profile_repository_data")
self.project_repository_data =
self.getProperty("project_repository_data")
- #self.repository = self.getProperty("repository")
- self.repository = 'gentoo'
+ self.repository_data = self.getProperty("repository_data")
self.project_data = self.getProperty("project_data")
self.project_path = yield os.path.join(self.repository_basedir,
self.project_repository_data['name'] + '.git')
- self.repository_path = yield os.path.join(self.repository_basedir,
self.repository + '.git')
+ self.repository_path = yield os.path.join(self.repository_basedir,
self.repository_data['name'] + '.git')
self.portage_path = yield os.path.join(self.project_path,
self.project_data['name'], 'etc/portage')
success = True
for x in [
os.path.join(self.repository_basedir,
self.profile_repository_data['name'] + '.git'),
self.project_path,
self.portage_path,
- os.path.join(self.portage_path, 'make.profile')
+ os.path.join(self.portage_path, 'make.profile'),
self.repository_path
# check the path of make.profile is project_data['profile']
]:
@@ -84,7 +87,7 @@ class CheckPathGentooCiProject(BuildStep):
self.setProperty("project_data", self.project_data, 'project_data')
self.setProperty("project_repository_data",
self.project_repository_data, 'project_repository_data')
self.setProperty("cpv_changes", self.getProperty("cpv_changes"),
'cpv_changes')
- self.setProperty("repository", self.repository, 'repository')
+ self.setProperty("repository_data", self.repository_data,
'repository_data')
return SUCCESS
class CheckProjectGentooCiProject(BuildStep):
@@ -110,7 +113,7 @@ class CheckProjectGentooCiProject(BuildStep):
self.setProperty("config_root", self.config_root, 'config_root')
self.setProperty("project_data", self.project_data, 'project_data')
self.setProperty("cpv_changes", self.getProperty("cpv_changes"),
'cpv_changes')
- self.setProperty("repository", self.getProperty("repository"),
'repository')
+ self.setProperty("repository_data",
self.getProperty("repository_data"), 'repository')
return SUCCESS
class CheckCPVGentooCiProject(BuildStep):
@@ -119,15 +122,12 @@ class CheckCPVGentooCiProject(BuildStep):
@defer.inlineCallbacks
def run(self):
- self.config_root = self.getProperty("config_root")
#self.cpv_changes = self.getProperty("cpv_changes")
self.cpv_changes = []
- self.project_data = self.getProperty("project_data")
self.cpv_changes.append('dev-lang/python-3.8')
self.cpv_changes.append('dev-python/prust-3.9')
print(self.cpv_changes)
- self.repository = self.getProperty("repository")
- print(self.repository)
+ print(self.getProperty("repository_data"))
# check if cpv_change is a string or a list
if isinstance(self.cpv_changes, list):
self.cpv_list = self.cpv_changes
@@ -150,9 +150,9 @@ class CheckCPVGentooCiProject(BuildStep):
updateSourceStamp=False,
set_properties={
'cpv' : cpv,
- 'config_root' : self.config_root,
- 'project_data' : self.project_data,
- 'repository' : self.repository
+ 'config_root' : self.getProperty("config_root"),
+ 'project_data' : self.getProperty("project_data"),
+ 'repository_data' :
self.getProperty("repository_data"),
}
)
)
