commit: e985f2cf2dfd2ab17c7ea7185e82c25c60deb780
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 30 21:43:02 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Oct 30 21:43:02 2024 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e985f2cf
Update steps to support Buildbot 4.1.0
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 2 +
buildbot_gentoo_ci/steps/bugs.py | 4 +-
buildbot_gentoo_ci/steps/clean.py | 16 +++-
buildbot_gentoo_ci/steps/logs.py | 136 ++++++++++++++++++++++++-----
buildbot_gentoo_ci/steps/package.py | 11 ++-
buildbot_gentoo_ci/steps/version.py | 9 +-
6 files changed, 148 insertions(+), 30 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index ab80e19..fffa281 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -172,6 +172,8 @@ def parse_build_log():
f.addStep(logs.setEmergeInfoLog())
# add package info to log and db
f.addStep(logs.setPackageInfoLog())
+ # save build steps log to a file
+ f.addStep(logs.writeStepLogsToFile())
# make the bugreport
f.addStep(logs.SetupBugReportSteps())
# set BuildStatus
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 52bd08d..efff7f5 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -18,7 +18,7 @@ from buildbot.plugins import util
from buildbot_gentoo_ci.utils.regex import stripQuotesAndMore, finishTitle
-from bugz.cli import check_bugz_token, login, list_bugs
+from bugz.cli import check_bugz_token, check_auth, list_bugs
from bugz.cli_argparser import make_arg_parser
from bugz.configfile import load_config
from bugz.settings import Settings
@@ -64,7 +64,7 @@ def search_bugz(args):
for key in params:
log_info(' {0:<20} = {1}'.format(key, params[key]))
- login(settings)
+ check_auth(settings)
result = settings.call_bz(settings.bz.Bug.search, params)['bugs']
diff --git a/buildbot_gentoo_ci/steps/clean.py
b/buildbot_gentoo_ci/steps/clean.py
index e4d8573..fb28973 100644
--- a/buildbot_gentoo_ci/steps/clean.py
+++ b/buildbot_gentoo_ci/steps/clean.py
@@ -10,6 +10,7 @@ from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
from buildbot.plugins import steps, util
+from buildbot_gentoo_ci.db.builds import _db2data_ProjectsBuilds
class SetupPropertys(BuildStep):
name = 'Setup propertys for clean db'
description = 'Running'
@@ -40,7 +41,8 @@ class SetupPropertys(BuildStep):
# get build_data for deleted_ebuild_data['uuid']
build_data_list = yield
self.gentooci.db.builds.getBuildsByVersionUuid(ebuild_data['uuid'])
print(build_data_list)
- for build_data in build_data_list:
+ for build_data_model in build_data_list:
+ build_data = _db2data_ProjectsBuilds(build_data_model)
if build_data['bug_id'] == 0:
print(f"Build : {build_data['id']} will be removed")
if build_data['buildbot_build_id'] != 0:
@@ -72,6 +74,11 @@ class RemoveEbuildFromDb(BuildStep):
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ build_data_list = yield
self.gentooci.db.builds.getBuildsByVersionUuid(self.uuid)
+ print(build_data_list)
+ for build_data_model in build_data_list:
+ build_data = _db2data_ProjectsBuilds(build_data_model)
+ yield self.gentooci.db.builds.removeBuild(build_data['id'])
yield self.gentooci.db.versions.removeVersionKeyword(self.uuid)
yield self.gentooci.db.versions.removeVersionMetadata(self.uuid)
yield self.gentooci.db.versions.removeVersion(self.uuid)
@@ -112,7 +119,12 @@ class RemoveBuildFromDb(BuildStep):
self.descriptionDone = f"BuildBot Build : {self.build_id} will be
removed"
bb_build_data = yield self.master.db.builds.getBuild(self.build_id)
# remove steps and logs
- yield self.master.db.logs.deleteLogChunks(self.build_id)
+ steps_data = yield self.master.db.steps.getSteps(self.build_id)
+ for step_data in steps_data:
+ yield self.master.db.logs.deleteLog(step_data['id'])
+ logs_data = yield self.master.db.logs.getLogs(step_data['id'])
+ for log_data in logs_data:
+ yield self.master.db.logs.deleteLogChunks(log_data['id'])
# remove propertys
yield self.master.db.builds.pruneBuildProperties(self.build_id)
# remove buildset_sourcestamps
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 6b8eba5..fdd0f3e 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -1,6 +1,9 @@
-# Copyright 2023 Gentoo Authors
+# Copyright 2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
import os
import re
import gzip
@@ -27,6 +30,39 @@ from buildbot.plugins import util
from buildbot_gentoo_ci.steps import master as master_steps
from buildbot_gentoo_ci.steps import bugs
from buildbot_gentoo_ci.utils.regex import stripQuotesAndMore, finishTitle
+from buildbot_gentoo_ci.db.versions import _db2data_Version
+from buildbot_gentoo_ci.db.packages import _db2data_Package
+
+from buildbot.db.logs import LogModel
+from buildbot.db.steps import StepModel
+
+def _db2data_Log(model: LogModel):
+ if model is None:
+ return None
+ return {
+ 'logid': model.id,
+ 'name': model.name,
+ 'slug': model.slug,
+ 'stepid': model.stepid,
+ 'complete': model.complete,
+ 'num_lines': model.num_lines,
+ 'type': model.type,
+ }
+
+def _db2data_Step(model: StepModel):
+ return {
+ 'stepid': model.id,
+ 'number': model.number,
+ 'name': model.name,
+ 'buildid': model.buildid,
+ 'started_at': model.started_at,
+ 'locks_acquired_at': model.locks_acquired_at,
+ 'complete_at': model.complete_at,
+ 'state_string': model.state_string,
+ 'results': model.results,
+ 'urls': [{'name': item.name, 'url': item.url} for item in model.urls],
+ 'hidden': model.hidden,
+ }
def PersOutputOfLogParser(rc, stdout, stderr):
build_summery_output = {}
@@ -103,12 +139,14 @@ class SetupPropertys(BuildStep):
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ print(f"Project_build_data:{self.getProperty('project_build_data')}")
project_data = yield
self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_build_data')['project_uuid'])
default_project_data = yield
self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
version_data = yield
self.gentooci.db.versions.getVersionByUuid(self.getProperty('project_build_data')['version_uuid'])
+ print(f"Version_data:{version_data}")
self.setProperty("project_data", project_data, 'project_data')
self.setProperty("default_project_data", default_project_data,
'default_project_data')
- self.setProperty("version_data", version_data, 'version_data')
+ self.setProperty("version_data", _db2data_Version(version_data),
'version_data')
self.setProperty("status", 'completed', 'status')
if isinstance(self.getProperty('faild_cpv'), str):
log_cpv = self.getProperty('faild_cpv')
@@ -119,21 +157,6 @@ class SetupPropertys(BuildStep):
self.descriptionDone = 'Runing log checker on ' + log_cpv
logsdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildnumber']))
self.setProperty("logsdir", logsdir, 'logsdir')
- # get steps/log id's from build
- build_log_steps_data = {}
- steps_data = yield
self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id'])
- for step in steps_data:
- log_data = None
- logs_data = yield self.master.db.logs.getLogs(step['id'])
- for log_info in logs_data:
- if log_info['name'] != 'property changes':
- log_data = log_info
- step_info = {}
- step_info['name'] = step['name']
- step_info['number'] = step['number']
- step_info['log_data'] = log_data
- build_log_steps_data[step['id']] = step_info
- self.setProperty("build_log_steps_data", build_log_steps_data,
'build_log_steps_data')
return SUCCESS
class SetupParserBuildLoger(BuildStep):
@@ -539,15 +562,32 @@ class writeStepLogsToFile(BuildStep):
@defer.inlineCallbacks
def run(self):
+ # get steps/log id's from build
+ build_log_steps_data = {}
+ steps_data = yield
self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id'])
+ for dbdict_step in steps_data:
+ step = _db2data_Step(dbdict_step)
+ logs_data = yield self.master.db.logs.getLogs(step['stepid'])
+ log_data = None
+ for dbdict_log_info in logs_data:
+ log_info = _db2data_Log(dbdict_log_info)
+ if log_info['name'] != 'property changes':
+ log_data = log_info
+ step_info = {}
+ step_info['name'] = step['name']
+ step_info['number'] = step['number']
+ step_info['log_data'] = log_data
+ build_log_steps_data[step['stepid']] = step_info
+ self.setProperty("build_log_steps_data", build_log_steps_data,
'build_log_steps_data')
buildsteplogfile_path = yield
os.path.join(self.getProperty('logsdir'), 'buildsteps.log.xz')
stepstologlist = ['Run emerge step build', 'Run emerge step build_1']
logs_texts = []
- for k, v in self.getProperty('build_log_steps_data').items():
+ for k, v in build_log_steps_data.items():
if v['name'] in stepstologlist:
logs_texts.append(f">>> BEGINING OF STEP: {v['name']}")
logs_texts.append('')
log_data = v['log_data']
- log_text = yield
self.master.db.logs.getLogLines(log_data['id'], 1, log_data['num_lines'])
+ log_text = yield
self.master.db.logs.getLogLines(log_data['logid'], 1, log_data['num_lines'])
for line in log_text.split('\n'):
if line.startswith('h BUILDMASTER='):
line = 'h BUILDMASTER=XXX.XXX.XXX.XXX'
@@ -568,12 +608,57 @@ class SetupBugReportSteps(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
+ @defer.inlineCallbacks
+ def getCCAndAssigned_to(self):
+ emails_dict = self.getProperty('emails')
+ packages_emails = emails_dict['packages_emails']
+ change_emails = emails_dict['change']
+ if self.getProperty('failed_version_data'):
+ change_id = self.getProperty('failed_version_data')['change_id']
+ else:
+ change_id = self.getProperty('version_data')['change_id']
+ change = yield self.master.db.changes.getChange(change_id)
+ stab_key = False
+ bug_mails = []
+ for word in ['Stabilize', 'stabilize', 'Keyword', 'keyword']:
+ if word in change['comments']:
+ stab_key = True
+ change_author = change_emails['authors'].split(' <')[1][:-1]
+ if '[email protected]' or '[email protected]' in
packages_emails:
+ if not change_author.endswith('@gentoo.org') and not stab_key:
+ bug_mails.append(change_author)
+ for package_email in packages_emails:
+ if not package_email['email'].endswith('@gentoo.org') and
package_email['email'] not in bug_mails:
+ bug_mails.append(package_email['email'])
+ for package_email in packages_emails:
+ if bug_mails == []:
+ if package_email['mail_type'] == 'project':
+ bug_mails.append(package_email['email'])
+ elif '[email protected]' or '[email protected]' not in
packages_emails:
+ if change_author in package_email:
+ bug_mails.append(change_author)
+ if not change_author.endswith('@gentoo.org') not in
packages_emails:
+ bug_mails.append(change_author)
+ for package_email in packages_emails:
+ if bug_mails == []:
+ if package_email['mail_type'] == 'project':
+ bug_mails.append(package_email['email'])
+ if change_author not in bug_mails and not stab_key:
+ bug_mails.append(change_author)
+ for package_email in packages_emails:
+ if not package_email['email'] in bug_mails:
+ bug_mails.append(package_email['email'])
+ print(f"bug_mails: {bug_mails}")
+ self.setProperty("bug_mails", bug_mails, 'bug_mails')
+ return bug_mails
+
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
bug_config = self.gentooci.config.project['bug_config']
make_bugreport = False
print(bug_config)
+ bug_mails = self.getCCAndAssigned_to()
if bug_config['enable'] and self.getProperty('status') == 'failed' and
self.getProperty('error_dict')['title_found'] and not self.getProperty('bgo'):
# add bug enable profile, repo and project
bug_settings_data = yield
self.gentooci.db.projects.getBugsSettingsByProjectUuid(self.getProperty("project_data")['uuid'])
@@ -581,9 +666,12 @@ class SetupBugReportSteps(BuildStep):
version_data = self.getProperty('failed_version_data')
else:
version_data = self.getProperty("version_data")
- package_data = yield
self.gentooci.db.packages.getPackageByUuid(version_data['package_uuid'])
- project_repository_settings = yield
self.gentooci.db.projects.getRepositorySettingByProjUuidAndRepoUuid(self.getProperty("project_data")['uuid'],
package_data['repository_uuid'])
- print(bug_settings_data)
+ print(version_data)
+ dbdict_package_data = yield
self.gentooci.db.packages.getPackageByUuid(version_data['package_uuid'])
+ package_data = _db2data_Package(dbdict_package_data)
+ print(package_data)
+ project_repository_settings = yield
self.gentooci.db.projects.getRepositorySettingByProjUuidAndRepoUuid(
+ self.getProperty("project_data")['uuid'],
package_data['repository_uuid'])
print(project_repository_settings)
if bug_settings_data['enabled'] and
project_repository_settings['bug']:
make_bugreport = True
@@ -643,6 +731,10 @@ class SetupBugReportSteps(BuildStep):
bug_params4['filename'] = yield
os.path.join(self.getProperty('logsdir'), filename)
#bug_params4['comment'] = filename
aftersteps_list.append(bugs.Attach(bug_args, bug_params4))
+ bug_params5 = {}
+ #bug_params5['assigned_to'] = assigned_to
+ #bug_params5['cc'] = cc_list
+ #aftersteps_list.append(bugs.Modify(bug_args, bug_params5))
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
return SKIPPED
diff --git a/buildbot_gentoo_ci/steps/package.py
b/buildbot_gentoo_ci/steps/package.py
index 847ba12..bd42a48 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -1,6 +1,9 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
import re
import os
@@ -16,6 +19,8 @@ from buildbot.process.results import SKIPPED
from buildbot.process import remotecommand
from buildbot.plugins import steps
+from buildbot_gentoo_ci.db.packages import _db2data_Package
+
class SetupPropertys(BuildStep):
name = 'Setup propertys for CPV check'
description = 'Running'
@@ -246,7 +251,7 @@ class CheckP(BuildStep):
self.setProperty("package", self.package, 'package')
yield self.build.addStepsAfterCurrentStep([AddPackage()])
else:
- self.setProperty("package_data", self.package_data, 'package_data')
+ self.setProperty("package_data",
_db2data_Package(self.package_data), 'package_data')
return SUCCESS
class TriggerCheckForV(BuildStep):
@@ -306,8 +311,10 @@ class TriggerCleanPackageDb(BuildStep):
days_in_s = 86400 * days
needcleaning = 0
now = int(self.master.reactor.seconds())
+ print(f"Now: {now} Days: {days_in_s} Now - Days: {now - days_in_s} in
s Old ebuilds: {len(deleted_ebuilds_data)}")
for deleted_ebuild_data in deleted_ebuilds_data:
# check if older the days
+ print(f"Ebuild deleted time in s:
{deleted_ebuild_data['deleted_at']} Days old: {divmod(now -
deleted_ebuild_data['deleted_at'], 86400)}")
if deleted_ebuild_data['deleted_at'] < (now - days_in_s):
needcleaning = needcleaning + 1
print(f"{str(needcleaning)} ebuilds need cleaning in db")
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index 5f252c7..56f107c 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -1,6 +1,9 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
import os
from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
@@ -17,6 +20,8 @@ from buildbot.process.results import SKIPPED
from buildbot.process import remotecommand
from buildbot.plugins import steps
+from buildbot_gentoo_ci.db.versions import _db2data_Version
+
def GetPythonVersion(pyprop):
py_list = pyprop.replace(' ', '').replace('\n ', '').split('.')
return '.'.join([py_list[0], py_list[1]]).lower()
@@ -402,7 +407,7 @@ class SetupPropertys(BuildStep):
print(self.version)
self.old_version_data = yield
self.gentooci.db.versions.getVersionByName(self.version,
self.getProperty("package_data")['uuid'])
print(self.old_version_data)
- self.setProperty("old_version_data", self.old_version_data,
'old_version_data')
+ self.setProperty("old_version_data",
_db2data_Version(self.old_version_data), 'old_version_data')
self.setProperty("version", self.version, 'version')
return SUCCESS
