[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 4707a4cb41619d8c35d2d9f7cb0cc5412f447492
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Mar 31 19:38:55 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Mar 31 19:38:55 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4707a4cb
Fix a typo on faild_dep
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 53f79ab..1db8cc2 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1084,7 +1084,7 @@ class CheckEmergeLogs(BuildStep):
'pkg_check_log_data' :
self.getProperty("pkg_check_log_data"),
'repository_data' :
self.getProperty('repository_data'),
'faild_cpv' : emerge_output['failed'],
-'faild_dep' : self.faild_dep
+'faild_dep' : self.faild_dep,
'build_workername' : self.getProperty('workername')
}
))
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: e8e5b67dc70b8b2b0222486574a20d6edb0577c2
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Mar 31 19:40:33 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Mar 31 19:40:33 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e8e5b67d
Add step logs to a log file
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 53 +---
1 file changed, 50 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 896778c..6b8eba5 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -7,6 +7,7 @@ import gzip
import io
import hashlib
import json
+import lzma
from portage.versions import catpkgsplit, cpv_getversion
@@ -74,6 +75,18 @@ def PersOutputOfEmergeInfo(rc, stdout, stderr):
'emerge_info_output' : emerge_info_output
}
+@defer.inlineCallbacks
+def WriteTextToFile(path, text_list, separator=False):
+separator2 = '\n'
+if separator:
+text_string = separator2.join(text_list)
+else:
+text_string = text_list
+text_string = text_string + separator2
+with lzma.open(path,"wt") as f:
+yield f.write(text_string)
+yield f.close
+
class SetupPropertys(BuildStep):
name = 'SetupPropertys'
@@ -109,14 +122,16 @@ class SetupPropertys(BuildStep):
# get steps/log id's from build
build_log_steps_data = {}
steps_data = yield
self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id'])
-print(steps_data)
for step in steps_data:
+log_data = None
logs_data = yield self.master.db.logs.getLogs(step['id'])
-print(logs_data)
+for log_info in logs_data:
+if log_info['name'] != 'property changes':
+log_data = log_info
step_info = {}
step_info['name'] = step['name']
step_info['number'] = step['number']
-step_info['log_data'] = logs_data
+step_info['log_data'] = log_data
build_log_steps_data[step['id']] = step_info
self.setProperty("build_log_steps_data", build_log_steps_data,
'build_log_steps_data')
return SUCCESS
@@ -509,6 +524,38 @@ class ParserPkgCheckLog(BuildStep):
returnstatus = WARNINGS
return returnstatus
+class writeStepLogsToFile(BuildStep):
+
+name = 'writeStepLogsToFile'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = False
+flunkOnFailure = True
+warnOnWarnings = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+buildsteplogfile_path = yield
os.path.join(self.getProperty('logsdir'), 'buildsteps.log.xz')
+stepstologlist = ['Run emerge step build', 'Run emerge step build_1']
+logs_texts = []
+for k, v in self.getProperty('build_log_steps_data').items():
+if v['name'] in stepstologlist:
+logs_texts.append(f">>> BEGINING OF STEP: {v['name']}")
+logs_texts.append('')
+log_data = v['log_data']
+log_text = yield
self.master.db.logs.getLogLines(log_data['id'], 1, log_data['num_lines'])
+for line in log_text.split('\n'):
+if line.startswith('h BUILDMASTER='):
+line = 'h BUILDMASTER=XXX.XXX.XXX.XXX'
+logs_texts.append(line[1:])
+logs_texts.append(f">>> END OF STEP: {v['name']}")
+yield WriteTextToFile(buildsteplogfile_path, logs_texts,
separator=True)
+return SUCCESS
+
class SetupBugReportSteps(BuildStep):
name = 'SetupBugReportSteps'
description = 'Running'
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/
commit: 99dcb588c27184023d01ae6a55308e999ce25c20
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Mar 30 09:16:33 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Mar 30 09:16:33 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=99dcb588
Add build dep irc report
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/reporters.py | 3 ++-
buildbot_gentoo_ci/steps/builders.py | 5 -
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/config/reporters.py
b/buildbot_gentoo_ci/config/reporters.py
index ede9178..c1f0aef 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -13,7 +13,8 @@ irc_template = '''{% set resultsList = ["\x0303SUCCESS",
"\x0308WARNINGS", "\x03
{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303"
}}Repo:{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
{{ build['properties']['revision'][0]|truncate(10, True) }} {{ "\x0302" }}{{
build['properties']['owners'][0][0] }}{{ "\x03" }} \
{{ "\x0306" }}{{ build['properties']['event'][0] }}{{ "\x03" }} {{ projects
}}:{{ build['properties']['project_data'][0]['name'] }} \
-{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }} \
+{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{
"\x02" }} Dep: {{ resultsList[build['properties']['faild_dep'][0]] }} \
+{{ "\x0312" }}{{ build_url }}{{ "\x03" }} \
{% if build['properties']['bgo'][0]['match'] is true %}{{ "\x0311" }}Bug: {{
"\x03" }}https://bugs.gentoo.org/{{build['properties']['bgo'][0]['id']}}{%
endif %}\
'''
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 73e82f5..53f79ab 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -899,6 +899,7 @@ class CheckEmergeLogs(BuildStep):
self.setProperty('faild_cpv', emerge_output['failed'], 'faild_cpv')
self.faild_version_data = False
self.log_data = {}
+self.faild_dep = 0
package_dict = emerge_output['packages']
#FIXME: Prosees the logs and do stuff
@@ -1060,6 +1061,7 @@ class CheckEmergeLogs(BuildStep):
self.faild_version_data =
self.getProperty("version_data")
else:
repository = False
+self.faild_dep = 2
for cpv, v in package_dict.items():
if cpv == emerge_output['failed']:
repository = v['repository']
@@ -1082,10 +1084,11 @@ class CheckEmergeLogs(BuildStep):
'pkg_check_log_data' :
self.getProperty("pkg_check_log_data"),
'repository_data' :
self.getProperty('repository_data'),
'faild_cpv' : emerge_output['failed'],
+'faild_dep' : self.faild_dep
'build_workername' : self.getProperty('workername')
}
))
-if rebuild:
+if isinstance(rebuild, str):
#FIXME: Set build timeout in config
build_timeout = 6600
shell_commad_list = []
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 16f9c0fc440bb4c15742b7e02827fad64d73db82
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Mar 23 14:13:36 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Mar 23 14:13:36 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=16f9c0fc
Redo Trigger for parse_build_log in CheckEmergeLogs
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 180 +--
1 file changed, 87 insertions(+), 93 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index c033196..73e82f5 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -895,8 +895,10 @@ class CheckEmergeLogs(BuildStep):
'-v'
]
emerge_output = self.getProperty('emerge_output')
-self.faild_cpv = emerge_output['failed']
+print(f"emerge_output: {emerge_output}")
+self.setProperty('faild_cpv', emerge_output['failed'], 'faild_cpv')
self.faild_version_data = False
+self.log_data = {}
package_dict = emerge_output['packages']
#FIXME: Prosees the logs and do stuff
@@ -916,7 +918,7 @@ class CheckEmergeLogs(BuildStep):
print(emerge_output)
# this should be set in the config
retrays = 5
-if self.getProperty('rerun') <= retrays and self.faild_cpv:
+if self.getProperty('rerun') <= retrays and
emerge_output['failed']:
# when we need to change use. we could rerun pre-build with
# --autounmask-use=y --autounmask-write=y --autounmask-only=y
# but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS
@@ -997,26 +999,24 @@ class CheckEmergeLogs(BuildStep):
self.aftersteps_list.append(CheckEmergeLogs('extra-build'))
else:
# trigger parse_build_log with info about pre-build and it fail
-if self.faild_cpv:
+if emerge_output['failed']:
pass
self.setProperty('rerun', 0, 'rerun')
# Make Logfile dict
if self.step == 'extra-build' or self.step == 'build':
-print(emerge_output)
log_dict = {}
# get cpv, logname and log path
for log_path in emerge_output['log_paths']:
c = log_path.split('/')[6]
full_logname = log_path.split('/')[7]
-print(full_logname)
+print(f"full_logname: {full_logname}")
pv = full_logname.split(':')[0]
cpv = c + '/' + pv
log_dict[cpv] = dict(
log_path = log_path,
full_logname = full_logname
)
-print('log_dict')
-print(log_dict)
+print(f"log_dict: {log_dict}")
if self.step == 'extra-build':
#FIXME: Check if extra build did work
self.aftersteps_list.append(RunEmerge(step='pre-build'))
@@ -1028,94 +1028,88 @@ class CheckEmergeLogs(BuildStep):
# local_log_path dir set in config
# format
/var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz
rebuild = False
-if self.step == 'build':
+if self.step == 'build' and log_dict != {}:
retrays = 1
-# Find log for cpv that was requested or did failed
-if not log_dict == {}:
-print('log_dict not emty')
-# requested cpv
-cpv = self.getProperty('cpv')
-if cpv in log_dict or self.faild_cpv in log_dict:
-print(cpv)
-print(self.faild_cpv)
+if isinstance(emerge_output['failed'], str):
+log_cpv = emerge_output['failed']
+if self.getProperty('rerun') < retrays:
+# set rebuild
+# haskell
+if c == 'dev-haskell':
+rebuild = 'haskell'
+# binary
+for cpv, v in package_dict.items():
+if cpv == emerge_output['failed']:
+if v['binary']:
+rebuild = 'binary'
+print(f"rebuild is: {rebuild}")
+else:
+log_cpv = self.getProperty('cpv')
+if not rebuild:
+if log_cpv in log_dict:
+print(f"Package {log_cpv} in log_dict")
+self.log_data[log_cpv] = log_dict[log_cpv]
yield self.createDistDir()
-if cpv in log_dict:
-self.log_data[cpv] = log_dict[cpv]
-yield self.getLogFile(cpv, log_dict)
-
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 6171182755e1b643be9205edae0a81d284bdab99
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Mar 21 16:56:34 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Mar 21 16:56:34 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=61711827
Add bb log id for build steps
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 9 +++--
buildbot_gentoo_ci/steps/logs.py | 15 ++-
2 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 0ac8526..c033196 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -347,13 +347,19 @@ class SetupPropertys(BuildStep):
self.setProperty('rerun', 0, 'rerun')
project_build_data = self.getProperty('project_build_data')
project_build_data['status'] = 'in-progress'
-project_build_data['buildbot_build_id'] =
self.getProperty("buildnumber")
yield self.gentooci.db.builds.setStatusBuilds(
project_build_data['id'],
project_build_data['status'])
+builder_id = yield
self.master.db.builders.findBuilderId(self.getProperty("buildername"),
autoCreate=False)
+print(builder_id)
+build_data = yield self.master.db.builds.getBuildByNumber(builder_id
,self.getProperty("buildnumber"))
+print(build_data)
+project_build_data['buildbot_build_id'] = build_data['id']
yield self.gentooci.db.builds.setBuildbotBuildIdBuilds(
project_build_data['id'],
project_build_data['buildbot_build_id'])
+project_build_data['builder_id'] = builder_id
+project_build_data['buildnumber'] = self.getProperty("buildnumber")
self.setProperty('project_build_data', project_build_data,
'project_build_data')
print(self.getProperty("project_build_data"))
self.masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber")))
@@ -1082,7 +1088,6 @@ class CheckEmergeLogs(BuildStep):
'pkg_check_log_data' :
self.getProperty("pkg_check_log_data"),
'repository_data' :
self.getProperty('repository_data'),
'faild_cpv' : self.faild_cpv,
-'step' : self.step,
'build_workername' :
self.getProperty('workername')
}
))
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index b6c97b7..896778c 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -104,8 +104,21 @@ class SetupPropertys(BuildStep):
self.setProperty("log_cpv", log_cpv, 'log_cpv')
self.setProperty("bgo", False, 'bgo')
self.descriptionDone = 'Runing log checker on ' + log_cpv
-logsdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildbot_build_id']))
+logsdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildnumber']))
self.setProperty("logsdir", logsdir, 'logsdir')
+# get steps/log id's from build
+build_log_steps_data = {}
+steps_data = yield
self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id'])
+print(steps_data)
+for step in steps_data:
+logs_data = yield self.master.db.logs.getLogs(step['id'])
+print(logs_data)
+step_info = {}
+step_info['name'] = step['name']
+step_info['number'] = step['number']
+step_info['log_data'] = logs_data
+build_log_steps_data[step['id']] = step_info
+self.setProperty("build_log_steps_data", build_log_steps_data,
'build_log_steps_data')
return SUCCESS
class SetupParserBuildLoger(BuildStep):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 33713a909627a0378845d9d0be4f689c3857667b
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Mar 19 17:18:49 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Mar 19 17:18:49 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=33713a90
Add compile to word_list as Ago use it as default
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 64bd3e7..b6c97b7 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -210,6 +210,8 @@ class MakeIssue(BuildStep):
cp = c + '/' + p
word_list.append(cp)
word_list.append(cpv_getversion(self.error_dict['cpv']))
+# add compile as Ago use that as default
+word_list.append('compile')
for word in self.error_dict['title_phase'].split(' '):
word_list.append(word)
for word in self.error_dict['title_issue_nice'].split(' '):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 6ca6f7fb23f0d63bbb53a457c1bd149a9cd40795
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Mar 17 20:40:25 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Mar 17 20:40:25 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6ca6f7fb
Add more support for change use, rebuild faild bin and clean some --change-*
emerge options
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 49
1 file changed, 39 insertions(+), 10 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 09739d7..0ac8526 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -31,6 +31,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['masked'] = False
emerge_output['failed'] = False
package_dict = {}
+change_use = {}
log_path_list = []
print('stderr')
print(stderr)
@@ -99,7 +100,6 @@ def PersOutputOfEmerge(rc, stdout, stderr):
change_use_list = []
# get cpv
cpv_split = line_list[1].split(':')
-change_use = {}
# add use flags
if line_list[4].startswith('+') or line_list[4].startswith('-'):
# we only support tre for now
@@ -120,7 +120,21 @@ def PersOutputOfEmerge(rc, stdout, stderr):
change_use_list = False
if change_use_list:
change_use[cpv_split[0]] = change_use_list
-emerge_output['change_use'] = change_use
+if 'USE changes are' in line:
+change_use_list = []
+change_use = {}
+for text in stderr.split('\n'):
+if text.startswith('>='):
+line_list = text.split(' ')
+for line2 in text.split(' '):
+if line2.startswith('>='):
+cpv = line2.split(':')[0].replace('>=', '')
+else:
+change_use_list.append(line2)
+if change_use_list:
+change_use[cpv] = change_use_list
+if change_use != {}:
+emerge_output['change_use'] = change_use
if line.startswith(' * '):
if line.endswith('.log'):
log_path = line.split(' ')[3]
@@ -355,8 +369,9 @@ class RunEmerge(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, step=None, **kwargs):
+def __init__(self, step=None, exclude=False, **kwargs):
self.step = step
+self.exclude = exclude
super().__init__(**kwargs)
self.descriptionSuffix = self.step
self.name = 'Setup emerge for ' + self.step + ' step'
@@ -367,6 +382,7 @@ class RunEmerge(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
projects_emerge_options = yield
self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
+#FIXME: Use settings from gentoo-ci-settings json file
self.stepname = 'Run emerge ' + 'step ' + self.step
shell_commad_list = [
'emerge',
@@ -402,9 +418,9 @@ class RunEmerge(BuildStep):
self.setProperty('build_env', build_env, 'build_env')
if self.step == 'pre-update':
-shell_commad_list.append('-uDN')
-shell_commad_list.append('--changed-deps')
-shell_commad_list.append('--changed-use')
+shell_commad_list.append('--update')
+shell_commad_list.append('--deep')
+shell_commad_list.append('--newuse')
shell_commad_list.append('--pretend')
shell_commad_list.append('@world')
# don't build bin for virtual and acct-*
@@ -423,9 +439,9 @@ class RunEmerge(BuildStep):
aftersteps_list.append(CheckEmergeLogs('pre-update'))
if self.step == 'update':
-shell_commad_list.append('-uDNq')
-shell_commad_list.append('--changed-deps')
-shell_commad_list.append('--changed-use')
+shell_commad_list.append('--update')
+shell_commad_list.append('--deep')
+shell_commad_list.append('--newuse')
shell_commad_list.append('@world')
# don't build bin for virtual and acct-*
shell_commad_list.append('--buildpkg-exclude')
@@ -600,6 +616,10 @@ class RunEmerge(BuildStep):
# we don't use the bin for the requsted cpv
shell_commad_list.append('--usepkg-exclude')
shell_commad_list.append(cp)
+if self.exclude:
+# we don't use the bin for the exclude cpv
+shell_commad_list.append('--usepkg-exclude')
+shell_commad_list.append(self.
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/utils/, buildbot_gentoo_ci/steps/
commit: ebac67eedbc65afaa3e3ae2abb22fba8a9277728
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Mar 16 10:09:12 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Mar 16 10:09:12 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ebac67ee
Add support for rules on title_line and fix regex on .patch
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 12 +++-
buildbot_gentoo_ci/utils/regex.py | 2 ++
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index a261dac..64bd3e7 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -216,6 +216,14 @@ class MakeIssue(BuildStep):
word_list.append(word)
return word_list
+def title_line_rules(self, text_issue_list):
+for text_issue in text_issue_list:
+# patch fail
+if re.search(' \* patch .* failed with /', text_issue):
+return text_issue.replace('*', '').strip()
+title_line = text_issue_list[0].replace('*', '').strip()
+return title_line
+
def ClassifyIssue(self):
# get the title for the issue
text_issue_list = []
@@ -228,7 +236,9 @@ class MakeIssue(BuildStep):
# add the issue error
self.error_dict['cpv'] = self.getProperty('log_cpv')
if text_issue_list != []:
-title_line = text_issue_list[0].replace('*', '').strip()
+print(f"text_issue_list: {text_issue_list}")
+title_line = self.title_line_rules(text_issue_list)
+print(f"title_line: {title_line}")
self.error_dict['title_issue'] = title_line
self.error_dict['title_issue_nice'] =
self.getNiceErrorLine(title_line)
self.error_dict['title_found'] = True
diff --git a/buildbot_gentoo_ci/utils/regex.py
b/buildbot_gentoo_ci/utils/regex.py
index c8d19d0..f9ed284 100644
--- a/buildbot_gentoo_ci/utils/regex.py
+++ b/buildbot_gentoo_ci/utils/regex.py
@@ -25,5 +25,7 @@ def finishTitle(word):
# ghyfv.v:78:9876: -> ghyfv.v
if word.endswith(':') and any(i.isdigit() for i in word):
word = word.split(':')[0]
+if word.startswith('/') and word.endswith('.patch'):
+word = word.split('/')[-1]
print(f"Word: {word2} Finish Word: {word}")
return word
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: c37dbde9c1e6688aa7bc9095fe64c91717d9c97d
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Mar 16 08:49:05 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Mar 16 08:49:05 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c37dbde9
Use faild_version_data when faild to get repo bug settings
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 16
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index f214ad6..a261dac 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -500,12 +500,20 @@ class SetupBugReportSteps(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
bug_config = self.gentooci.config.project['bug_config']
-bug_settings_data = yield
self.gentooci.db.projects.getBugsSettingsByProjectUuid(self.getProperty("project_data")['uuid'])
-project_repository_settings = yield
self.gentooci.db.projects.getRepositorySettingByProjUuidAndRepoUuid(self.getProperty("project_data")['uuid'],
self.getProperty('repository_data')['uuid'])
-# add bug enable profile and repo
make_bugreport = False
+print(bug_config)
if bug_config['enable'] and self.getProperty('status') == 'failed' and
self.getProperty('error_dict')['title_found'] and not self.getProperty('bgo'):
-if bug_settings_data['enable'] and
project_repository_settings['bug']:
+# add bug enable profile, repo and project
+bug_settings_data = yield
self.gentooci.db.projects.getBugsSettingsByProjectUuid(self.getProperty("project_data")['uuid'])
+if self.getProperty('failed_version_data'):
+version_data = self.getProperty('failed_version_data')
+else:
+version_data = self.getProperty("version_data")
+package_data = yield
self.gentooci.db.packages.getPackageByUuid(version_data['package_uuid'])
+project_repository_settings = yield
self.gentooci.db.projects.getRepositorySettingByProjUuidAndRepoUuid(self.getProperty("project_data")['uuid'],
package_data['repository_uuid'])
+print(bug_settings_data)
+print(project_repository_settings)
+if bug_settings_data['enabled'] and
project_repository_settings['bug']:
make_bugreport = True
if make_bugreport:
aftersteps_list = []
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/
commit: 0693e5f1e3191f2b6ecb3938610731c745d69f4a
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Mar 14 21:00:44 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Mar 14 21:00:44 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0693e5f1
Get faild_version_data when faild and add more info in find bugs
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/packages.py| 3 ++-
buildbot_gentoo_ci/db/versions.py| 3 ++-
buildbot_gentoo_ci/steps/bugs.py | 16 +---
buildbot_gentoo_ci/steps/builders.py | 23 +++
buildbot_gentoo_ci/steps/logs.py | 16 +++-
5 files changed, 43 insertions(+), 18 deletions(-)
diff --git a/buildbot_gentoo_ci/db/packages.py
b/buildbot_gentoo_ci/db/packages.py
index 1f68637..ad4a244 100644
--- a/buildbot_gentoo_ci/db/packages.py
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -27,12 +27,13 @@ from buildbot.db import base
class PackagesConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
-def getPackageByName(self, name, c_uuid, repo_uuid):
+def getPackageByName(self, name, c_uuid, repo_uuid, deleted=False):
def thd(conn):
tbl = self.db.model.packages
q = tbl.select()
q = q.where(tbl.c.name == name)
q = q.where(tbl.c.category_uuid == c_uuid)
+q = q.where(tbl.c.deleted == deleted)
q = q.where(tbl.c.repository_uuid == repo_uuid)
res = conn.execute(q)
row = res.fetchone()
diff --git a/buildbot_gentoo_ci/db/versions.py
b/buildbot_gentoo_ci/db/versions.py
index 8d35602..afb9e52 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -26,12 +26,13 @@ from buildbot.db import base
class VersionsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
-def getVersionByName(self, name, p_uuid):
+def getVersionByName(self, name, p_uuid, deleted=False):
def thd(conn):
tbl = self.db.model.versions
q = tbl.select()
q = q.where(tbl.c.name == name)
q = q.where(tbl.c.package_uuid == p_uuid)
+q = q.where(tbl.c.deleted == deleted)
res = conn.execute(q)
row = res.fetchone()
if not row:
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 6a097d4..52bd08d 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -123,26 +123,36 @@ class GetBugs(BuildStep):
# we splite the lines to lists and try to match the words
matches = 0
bug_words = list(bug['summary'].split())
+# add c/p and v
+for word in bug_words:
+if pkgsplit(word) is not None:
+ bug_words.append(pkgsplit(word)[0])
+ bug_words.append(cpv_getversion(word))
print(bug_words)
print(self.getProperty('error_dict')['words'])
for word in bug_words:
# clean the word
word = finishTitle(stripQuotesAndMore(word))
-print(word)
for match_word in self.getProperty('error_dict')['words']:
-if match_word in word:
+if match_word == word:
matches = matches + 1
print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
Summary: {bug['summary']}")
-if matches >= len(self.getProperty('error_dict')['title_nice']) -
self.bug_config['min_match_word']:
+len_match_words = len(self.getProperty('error_dict')['words']) -
self.bug_config['min_match_word']
+print(f"Min words to matchs: {str(len_match_words)}")
+if matches >= len_match_words or matches >=
self.bug_config['max_match_word']:
match = {}
match['match'] = True
match['id'] = bug['id']
match['summary'] = bug['summary']
yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue']}\n")
+yield log.addStdout(f"Nice line to match:
{self.getProperty('error_dict')['title_nice']}\n")
+yield log.addStdout(f"Words to match:
{self.getProperty('error_dict')['words']}\n")
if match:
yield log.addStdout('Match bug: YES\n')
yield log.addStdout(f"Bug: {str(match['id'])} Summary:
{match['summary']}\n")
self.setProperty("bgo", match, 'bgo')
+print(f"Match bug: YES")
+print(f"Bug: {str(match['id'])} Summary: {match['summary']}")
return match
yield log.addStdout('Match bug: NO\n')
return match
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 117fb70..09739d7 100644
--- a/buildbot_gentoo_ci/steps/builders.py
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/utils/
commit: 955e80294c70d1a47ab39b656e9d9357cf3f5f8d
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Mar 12 20:39:02 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Mar 12 20:39:02 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=955e8029
Fix finishTitle for path regex
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/utils/regex.py | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/utils/regex.py
b/buildbot_gentoo_ci/utils/regex.py
index 29372e0..c8d19d0 100644
--- a/buildbot_gentoo_ci/utils/regex.py
+++ b/buildbot_gentoo_ci/utils/regex.py
@@ -5,18 +5,25 @@ import re
# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L12
def stripQuotesAndMore(word):
+word2 = word
word = re.sub(r"b'", "", word)
word = re.sub(r"'", "", word)
word = re.sub(r'`', '', word)
word = re.sub(r'"', '', word)
word = re.sub(r'\\', '', word)
+print(f"Word: {word2} Finish Word: {word}")
return word
# strip away hex addresses, loong path names, line and time numbers and other
stuff
# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L469
# FIXME: Add the needed line when needed
def finishTitle(word):
-if word.startswith('/'):
+word2 = word
+# /usr/foo/baa/ghyk.c:67:69: -> ghyk.c:567:76:
+if '/' in word and word.endswith(':'):
word = word.split('/')[-1]
-word = re.sub(":\d+:\d+:", "", word)
+# ghyfv.v:78:9876: -> ghyfv.v
+if word.endswith(':') and any(i.isdigit() for i in word):
+word = word.split(':')[0]
+print(f"Word: {word2} Finish Word: {word}")
return word
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
commit: d8186f7796fbd57c68beafeb53978d3a2d20fc23
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Mar 3 07:53:29 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Mar 3 07:53:29 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d8186f77
Add bugsettings to project
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/model.py| 13 +++
buildbot_gentoo_ci/db/projects.py | 41 +-
buildbot_gentoo_ci/steps/logs.py | 8 -
sql/gentoo_ci_schema.sql | 71 ++-
4 files changed, 130 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index dea3e15..448e67f 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -152,6 +152,18 @@ class Model(base.DBConnectorComponent):
nullable=False),
)
+# projects bugs settings
+projects_bugs = sautils.Table(
+"projects_bugs", metadata,
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+sa.Column('enabled', sa.Boolean, default=False),
+sa.Column('auto_assigne', sa.Boolean, default=False),
+sa.Column('extra_summery', sa.String(255), nullable=False),
+)
+
# What repository's use by projects
projects_repositorys = sautils.Table(
"projects_repositorys", metadata,
@@ -167,6 +179,7 @@ class Model(base.DBConnectorComponent):
sa.Column('build', sa.Boolean, default=False),
sa.Column('test', sa.Boolean, default=False),
sa.Column('test_mr', sa.Boolean, default=False),
+sa.Column('bug', sa.Boolean, default=False),
)
# projects etc/portage settings
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 7ffac97..23cabde 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -79,6 +79,21 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+@defer.inlineCallbacks
+def getRepositorySettingByProjUuidAndRepoUuid(self, project_uuid,
repository_uuid):
+def thd(conn):
+tbl = self.db.model.projects_repositorys
+q = tbl.select()
+q = q.where(tbl.c.project_uuid == project_uuid)
+q = q.where(tbl.c.repository_uuid == repository_uuid)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict_projects_repositorys(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
@defer.inlineCallbacks
def getRepositorysByProjectUuid(self, uuid, auto=True):
def thd(conn):
@@ -212,6 +227,20 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+@defer.inlineCallbacks
+def getBugsSettingsByProjectUuid(self, uuid):
+def thd(conn):
+tbl = self.db.model.projects_bugs
+q = tbl.select()
+q = q.where(tbl.c.project_uuid == uuid)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict_projects_bugs(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -243,7 +272,8 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
pkgcheck=pkgcheck,
build=row.build,
test=row.test,
-test_mr=row.test_mr
+test_mr=row.test_mr,
+bug=row.bug
)
def _row2dict_projects_workers(self, conn, row):
@@ -312,3 +342,12 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
status=row.status,
type=row.type
)
+
+def _row2dict_projects_bugs(self, conn, row):
+return dict(
+id=row.id,
+project_uuid=row.project_uuid,
+enabled=row.enabled,
+auto_assigne=row.auto_assigne,
+extra_summery=row.extra_summery
+)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 34aee29..84f5058 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -494,8 +494,14 @@ class SetupBugReportSteps(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
bug_config = self.gentooci.config.project['bug_config']
+bug_settings_data = yield
self.gentooci.db.projects.getBugsSettingsByProjectUuid(self.get
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, sql/, ...
commit: 4269430378d05d5161d87056caf6817890d44493
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Feb 29 20:52:42 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Feb 29 20:52:42 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=42694303
Make emails dict for bugreports
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/schedulers.py | 5 +++--
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/db/versions.py | 6 --
buildbot_gentoo_ci/steps/logs.py| 26 +++---
buildbot_gentoo_ci/steps/version.py | 4 +++-
sql/gentoo_ci_schema.sql| 1 +
6 files changed, 35 insertions(+), 8 deletions(-)
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index c34d714..a4180cc 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -40,19 +40,20 @@ def getGitChanges(props):
change_data['timestamp'] = k['when_timestamp']
change_data['branch'] = k['branch']
change_data['project'] = k['project']
+change_data['change_id'] = k['changeid']
return change_data
def bb_branch_fn(branch):
# check branch
print(f"Branch: {branch}")
-if branch.endswith('-mr') or branch.endswith('-pr') or branch == 'master':
+if branch.endswith('-mr') or branch.endswith('-pr') or
branch.endswith('master') or branch.endswith('main'):
return True
return False
def bb_category_fn(category):
# check event
print(f"Category: {category}")
-if category == 'push' or category == 'merge_request':
+if category == 'push' or category == 'merge_request' or category is None:
return True
return False
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index b6a0c43..dea3e15 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -348,6 +348,7 @@ class Model(base.DBConnectorComponent):
nullable=False),
sa.Column('file_hash', sa.String(255), nullable=False),
sa.Column('commit_id', sa.String(255), nullable=False),
+sa.Column('change_id', sa.Integer, nullable=True, default=0),
sa.Column('deleted', sa.Boolean, default=False),
sa.Column('deleted_at', sa.Integer, nullable=True),
)
diff --git a/buildbot_gentoo_ci/db/versions.py
b/buildbot_gentoo_ci/db/versions.py
index 0aaac1b..8d35602 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -55,7 +55,7 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
return res
@defer.inlineCallbacks
-def addVersion(self, name, package_uuid, file_hash, commit_id):
+def addVersion(self, name, package_uuid, file_hash, commit_id, change_id):
def thd(conn, no_recurse=False):
try:
tbl = self.db.model.versions
@@ -63,7 +63,8 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
r = conn.execute(q, dict(name=name,
package_uuid=package_uuid,
file_hash=file_hash,
- commit_id=commit_id))
+ commit_id=commit_id,
+ change_id=change_id))
except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
uuid = None
else:
@@ -178,6 +179,7 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
package_uuid=row.package_uuid,
file_hash=row.file_hash,
commit_id=row.commit_id,
+change_id = row.change_id,
deleted=row.deleted,
deleted_at=row.deleted_at
)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index aacb8a5..34aee29 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -250,6 +250,7 @@ class MakeIssue(BuildStep):
self.summary_log_list = []
self.error_dict = {}
self.aftersteps_list = []
+Maintainers = []
#self.error_dict['hash'] = hashlib.sha256()
for k, v in sorted(self.summary_log_dict.items()):
self.summary_log_list.append(v['text'])
@@ -263,11 +264,30 @@ class MakeIssue(BuildStep):
self.error_dict['phase'] = v['text'].split(' (')[1].split('
phase')[0]
error = True
if v['text'].startswith(' * Maintainer:'):
-Maintainers = []
for email in v['text'].split(':')[1].split(' '):
if email != '':
-Maintainers.append(email)
-self.setProperty("Maintainers", Maintainers, 'Maintainers')
+for e in email.split(','):
+
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, py/
commit: 92b796f7c2580f9bd93a225072d56cbf8f13deaa
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Feb 22 23:58:06 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Feb 22 23:58:06 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=92b796f7
Fix phase_error and pattern_infos
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 14 ++
py/log_parser.py | 4 ++--
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 50e3f3f..aacb8a5 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -46,9 +46,10 @@ def PersOutputOfLogParser(rc, stdout, stderr):
'status' :
s['status'],
'line' : s['line'],
'search_pattern' :
s['search_pattern'],
-'phase' :
s['phase'],
+'phase' :
s['phase'].lower(),
'uuid' : s['uuid'],
'description' :
s['description'],
+'url' : s['url'],
})
build_summery_output['summary_log_dict'] = summary_log_dict
#FIXME: Handling of stderr output
@@ -221,7 +222,7 @@ class MakeIssue(BuildStep):
for k, v in sorted(self.summary_log_dict.items()):
# get the issue error
for s in v['pattern_infos']:
-if s['search_type'] == self.error_dict['phase'] and
s['status'] == 'error':
+if s['phase'] == self.error_dict['phase'] and s['status'] ==
'error':
text_issue_list.append(v['text'])
# add the issue error
self.error_dict['cpv'] = self.getProperty('log_cpv')
@@ -259,9 +260,14 @@ class MakeIssue(BuildStep):
# check if the build did fail
if v['text'].startswith(' * ERROR:') and v['text'].endswith('
phase):'):
# get phase error
-phase_error = v['text'].split(' (')[1].split(' phase')[0]
-self.error_dict['phase'] = phase_error
+self.error_dict['phase'] = v['text'].split(' (')[1].split('
phase')[0]
error = True
+if v['text'].startswith(' * Maintainer:'):
+Maintainers = []
+for email in v['text'].split(':')[1].split(' '):
+if email != '':
+Maintainers.append(email)
+self.setProperty("Maintainers", Maintainers, 'Maintainers')
#FIXME: write summary_log_list to a file
# add issue/bug/pr report
if error:
diff --git a/py/log_parser.py b/py/log_parser.py
index e3725bb..e8f319b 100644
--- a/py/log_parser.py
+++ b/py/log_parser.py
@@ -28,8 +28,8 @@ def get_pattern_dict(catchissue, i, uuid):
patten_dict['end'] = catchissue[2]
patten_dict['status'] = catchissue[3]
patten_dict['search_type'] = catchissue[4]
-patten_dict['url'] = catchissue[5]
-patten_dict['description'] = catchissue[6]
+patten_dict['url'] = catchissue[6]
+patten_dict['description'] = catchissue[7]
return patten_dict
def addPatternToList(phaseList, log_search_patterns, uuid):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, py/
commit: 6d8986da642b5d3b5670be40d8fe8883086c7678
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Feb 21 21:15:49 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Feb 21 21:15:49 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6d8986da
Move search pattern db to settings repository
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 34 ---
py/log_parser.py | 206 ---
2 files changed, 127 insertions(+), 113 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 85017a5..50e3f3f 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -42,10 +42,13 @@ def PersOutputOfLogParser(rc, stdout, stderr):
}
for s in v['pattern_info']:
summary_log_dict[int(k)]['pattern_infos'].append({
-'type' : s['type'],
+'search_type' :
s['search_type'],
'status' :
s['status'],
-'id' : s['id'],
+'line' : s['line'],
'search_pattern' :
s['search_pattern'],
+'phase' :
s['phase'],
+'uuid' : s['uuid'],
+'description' :
s['description'],
})
build_summery_output['summary_log_dict'] = summary_log_dict
#FIXME: Handling of stderr output
@@ -118,14 +121,23 @@ class SetupParserBuildLoger(BuildStep):
@defer.inlineCallbacks
def run(self):
+self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.aftersteps_list = []
log_cpv =
self.getProperty('log_build_data')[self.getProperty('log_cpv')]
build_log_file_compressed = log_cpv['full_logname'] + '.xz'
mastersrc_log = yield os.path.join(self.getProperty('logsdir'),
build_log_file_compressed)
log_py = 'log_parser.py'
-config_log_py = 'logparser.json'
mastersrc_py = yield os.path.join(self.master.basedir, log_py)
-mastersrc_config = yield os.path.join(self.master.basedir,
config_log_py)
+repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty("project_data")['settings_repository_uuid'])
+# Git clone settings repo
+
self.aftersteps_list.append(steps.GitLab(repourl=repository_data['url'],
+name = 'RunGit',
+descriptionDone=repository_data['name'],
+mode=repository_data['mode'],
+method=repository_data['method'],
+submodules=True,
+alwaysUseLatest=repository_data['alwaysuselatest']
+))
# Upload logfile to worker
self.aftersteps_list.append(steps.FileDownload(
mastersrc=mastersrc_log,
@@ -136,11 +148,6 @@ class SetupParserBuildLoger(BuildStep):
mastersrc=mastersrc_py,
workerdest=log_py
))
-# Upload log parser py config
-self.aftersteps_list.append(steps.FileDownload(
-mastersrc=mastersrc_config,
-workerdest=config_log_py
-))
#Untar the log
shell_commad_list = []
shell_commad_list.append('xz')
@@ -159,8 +166,11 @@ class SetupParserBuildLoger(BuildStep):
command.append(log_cpv['full_logname'])
command.append('-u')
command.append(self.getProperty('project_data')['uuid'])
-command.append('-d')
-command.append(util.Secret("log_parser_database"))
+command.append('-c')
+#FIXME: set it by images/flavors
+command.append('8')
+#FIXME: debug
+#command.append('-d')
self.aftersteps_list.append(steps.SetPropertyFromCommand(
name =
'RunBuildLogParser',
haltOnFailure =
True,
@@ -211,7 +221,7 @@ class MakeIssue(BuildStep):
for k, v in sorted(self.summary_log_dict.items()):
# get the issue error
for s in v['pattern_infos']:
-if s['type'] == self.error_dict['phase'] and s[
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 7f25300ddcf94e1d5018e9bc343a22155e3b043c
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Feb 5 18:54:20 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Feb 5 18:54:20 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7f25300d
Use settings repository for dockerfiles
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 34 ++
1 file changed, 14 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/nodes.py
b/buildbot_gentoo_ci/steps/nodes.py
index 7a01cbb..c1928ba 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -77,30 +77,26 @@ class SetupBuildStepsForDocker(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
node_data = self.getProperty("node_data")
image_data = self.getProperty("image_data")
-dockerfile = self.getProperty("image_data")['dockerfile'] +
'.Dockerfile'
+dockerfile = os.path.join('Project',
self.getProperty("project_data")['uuid'],'docker', image_data['dockerfile'] +
'.Dockerfile')
+repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty("project_data")['settings_repository_uuid'])
dockerfile_path = os.path.join(self.master.basedir, 'docker')
#FIXME: Set build timeout in config
self.build_timeout = 6600
aftersteps_list = []
-aftersteps_list.append(steps.FileDownload(
-flunkOnFailure=True,
-name='Upload Dockerfile',
-mastersrc = os.path.join(dockerfile_path, dockerfile),
-workerdest = dockerfile
-))
+aftersteps_list.append(steps.GitLab(repourl=repository_data['url'],
+name = 'RunGit',
+descriptionDone=repository_data['name'],
+mode=repository_data['mode'],
+method=repository_data['method'],
+submodules=True,
+alwaysUseLatest=repository_data['alwaysuselatest']
+))
# build the gentoo docker buildbot-worker image
# set the build args
self.buildargs = {}
-# set PROJECTUUID
-self.buildargs['PROJECTUUID'] = self.getProperty("project_uuid")
-# set ACCEPT_KEYWORDS
-keyword_data = yield
self.gentooci.db.keywords.getKeywordById(self.getProperty('project_data')['keyword_id'])
-if self.getProperty('project_data')['status'] == 'unstable':
-self.buildargs['KEYWORDS'] = '~' + keyword_data['name']
-else:
-self.buildargs['KEYWORDS'] = keyword_data['name']
-# set ARCH
-self.buildargs['ARCH'] = keyword_data['name']
+# set PROJECTUUID and PROJECTNAME
+self.buildargs['PROJECTUUID'] =
self.getProperty("project_data")['uuid']
+self.buildargs['PROJECTNAME'] =
self.getProperty("project_data")['name']
# set MAKEOPTS
# set USE
# set CFLAGS
@@ -132,12 +128,10 @@ class SetupBuildStepsForDocker(BuildStep):
flunkOnFailure=True,
name='Build docker buildbot-worker image',
command=['docker',
- 'buildx',
'build',
'--file', dockerfile,
'--build-arg', 'PROJECTUUID='+
self.buildargs['PROJECTUUID'],
- '--build-arg', 'KEYWORDS=' +
self.buildargs['KEYWORDS'],
- '--build-arg', 'ARCH=' +
self.buildargs['ARCH'],
+ '--build-arg', 'PROJECTNAME='+
self.buildargs['PROJECTNAME'],
'--build-arg', 'BINHOSTURL=' +
self.buildargs['BINHOSTURL'],
'--build-arg', 'BOOTSTRAPTAG=' +
self.buildargs['BOOTSTRAPTAG'],
'--tag', image_data['tag'],
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 34ce6c4b4c8420b77870ef157c60984dc5ae46ce
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Feb 4 11:32:24 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Feb 4 11:32:24 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=34ce6c4b
Use project_data for profile link
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 775e076..f90da5a 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -92,12 +92,12 @@ class SetMakeProfile(BuildStep):
portage_repos_path = self.getProperty('portage_repos_path')
project_data = self.getProperty('project_data')
profile_repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
-makeprofiles_paths = []
+#makeprofiles_paths = []
#NOTE: pkgcheck don't support make.profile as a dir
# we only support one line in db
-makeprofiles_data = yield
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'],
'make.profile')
-for makeprofile in makeprofiles_data:
-makeprofile_path = yield os.path.join(portage_repos_path,
profile_repository_data['name'], 'profiles', makeprofile['value'], '')
+#makeprofiles_data = yield
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'],
'make.profile')
+#for makeprofile in makeprofiles_data:
+makeprofile_path = yield os.path.join(portage_repos_path,
profile_repository_data['name'], 'profiles', project_data['profile'], '')
#makeprofiles_paths.append('../../..' + makeprofile_path)
#separator = '\n'
#makeprofile_path_string = separator.join(makeprofiles_paths)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: aebeb68573134f1f1879b7513d5051e562dd3b1c
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Feb 4 09:49:24 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Feb 4 09:49:24 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aebeb685
Use flavor to set MAKEOPTS -j*
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 74a8e37..775e076 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -228,7 +228,12 @@ class SetHostConf(BuildStep):
separator2 = ' '
makeconf_list = []
log = yield self.addLog('host.conf')
-makeconf_list.append('MAKEOPTS="-j14"')
+# set MAKEOPTS from flavor
+worker_data = yield
self.gentooci.db.workers.getWorkerByUuid(self.getProperty('workername'))
+print('worker_data')
+worker_flavor = yield
self.gentooci.db.workers.getFlavorById(worker_data['flavor_id'])
+print('worker_flavor')
+makeconf_list.append('MAKEOPTS="-j' + str(worker_flavor['cpu']) + '"')
makeconf_string = separator1.join(makeconf_list)
print(makeconf_string)
yield self.build.addStepsAfterCurrentStep([
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, ...
commit: 8449a45df3f33ccf90f60540a5aae1b571e3fe41
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Feb 3 10:02:26 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Feb 3 10:02:26 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8449a45d
Use settings repository for portage settings
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 33 +++
buildbot_gentoo_ci/db/model.py | 5 +-
buildbot_gentoo_ci/db/projects.py | 3 +-
buildbot_gentoo_ci/db/repositorys.py | 2 +-
buildbot_gentoo_ci/steps/portage.py| 89 +++---
buildbot_gentoo_ci/steps/repos.py | 4 +-
sql/gentoo_ci_schema.sql | 1 +
7 files changed, 60 insertions(+), 77 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 6755308..ab80e19 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from buildbot.plugins import steps as buildbot_steps
@@ -105,32 +105,13 @@ def run_build_request():
# update the repositorys listed in project_repository
f.addStep(repos.UpdateRepos())
# Clean and add new /etc/portage
-#NOTE: remove the symlink befor the dir
-#f.addStep(buildbot_steps.ShellCommand(
-#flunkOnFailure=False,
-#name='Clean make.profile',
-#command=['rm', 'make.profile'],
-#workdir='/etc/portage/'
-#))
-if buildbot_steps.FileExists(file='portage/make.conf', workdir='/etc/',
haltOnFailure = False):
-f.addStep(buildbot_steps.ShellCommand(
+f.addStep(buildbot_steps.ShellCommand(
flunkOnFailure=False,
name='Remove portage dir',
command=['rm', '-R', 'portage'],
workdir='/etc/'
))
-f.addStep(buildbot_steps.ShellCommand(
-flunkOnFailure=False,
-name='Create portage dir',
-command=['mkdir', 'portage'],
-workdir='/etc/'
-))
-#f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
-#name='Remove portage dir',
-#workdir='/etc/'))
-#f.addStep(buildbot_steps.MakeDirectory(dir="portage",
-#name = 'Create the portage dir',
-#workdir='/etc/'))
+f.addStep(portage.SetPortageSettings())
# Clean /var/cache/portage/logs and emerge.log
f.addStep(buildbot_steps.ShellCommand(
flunkOnFailure=False,
@@ -153,13 +134,13 @@ def run_build_request():
# setup repos.conf dir
#f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
#workdir='/etc/portage/'))
-f.addStep(portage.SetReposConf())
+#f.addStep(portage.SetReposConf())
# setup make.conf
-f.addStep(portage.SetMakeConf())
+f.addStep(portage.SetHostConf())
# setup env
-f.addStep(portage.SetEnvDefault())
+#f.addStep(portage.SetEnvDefault())
# setup package.*
-f.addStep(portage.SetPackageDefault())
+#f.addStep(portage.SetPackageDefault())
# setup files in /etc if needed
# run --regen if needed on repo
# check cpv match
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 3ae4f92..b6a0c43 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -15,7 +15,7 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.model.py
# Modifyed by Gentoo Authors.
-# Copyright 2023 Gentoo Authors
+# Copyright 2024 Gentoo Authors
import uuid
@@ -136,6 +136,9 @@ class Model(base.DBConnectorComponent):
sa.Column('profile_repository_uuid', sa.String(36),
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
nullable=False),
+sa.Column('settings_repository_uuid', sa.String(36),
+ sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
+ nullable=False),
sa.Column('keyword_id', sa.Integer,
sa.ForeignKey('keywords.id', ondelete='CASCADE'),
nullable=False),
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 9dcfa45..7ffac97 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -15,7 +15,7 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.*
# Modifyed by Gentoo Authors.
-# Copyright 2021 Gentoo Authors
+# Copyright 202
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: eaf908437029fb291b327db680d550fbe36c86df Author: Magnus Granberg gentoo org> AuthorDate: Sat Feb 3 09:48:17 2024 + Commit: Magnus Granberg gentoo org> CommitDate: Sat Feb 3 09:48:17 2024 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=eaf90843 Remove minio Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/minio.py | 60 --- 1 file changed, 60 deletions(-) diff --git a/buildbot_gentoo_ci/steps/minio.py b/buildbot_gentoo_ci/steps/minio.py deleted file mode 100644 index ac9ac3d..000 --- a/buildbot_gentoo_ci/steps/minio.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -from minio import Minio -from minio.error import ResponseError, BucketAlreadyOwnedByYou, BucketAlreadyExists - -from twisted.internet import defer -from twisted.python import log - -from buildbot.process.buildstep import BuildStep -from buildbot.process.results import SUCCESS -from buildbot.process.results import FAILURE - -#FIXME: -# get url, user from config -# get password from secret -url = '' -user = '' -password = '' - -class putFileToMinio(BuildStep): - -name = 'putFileToMinio' -description = 'Running' -descriptionDone = 'Ran' -descriptionSuffix = None -haltOnFailure = False -flunkOnFailure = True -warnOnWarnings = True - -def __init__(self, filename, target, bucket, **kwargs): -self.filename = filename -self.bucket = bucket -self.target = target -super().__init__(**kwargs) - -def getMinioConnect(self, url, user, password): -minioclient = Minio( -url, -access_key = user, -secret_key = password, -secure = False -) -return minioclient - -@defer.inlineCallbacks -def pushFileToMinio(self): -try: -yield self.minio_connect.fput_object(self.bucket, self.target, self.filename) -except ResponseError as err: -print(err) -return FAILURE -return True - -@defer.inlineCallbacks -def run(self): -self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] -self.minio_connect = yield self.getMinioConnect(url, user, password) -success = yield self.pushFileToMinio() -return SUCCESS
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/
commit: eb2f05efcf0392a17894233de29b5c3ede8e1b43
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Jan 6 22:27:24 2024 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Jan 6 22:27:24 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=eb2f05ef
Add getGitlabContext to Gitlab status
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/reporters.py | 21 +++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/config/reporters.py
b/buildbot_gentoo_ci/config/reporters.py
index 54472b0..ede9178 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -1,6 +1,8 @@
# Copyright 2022 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from twisted.internet import defer
+
from buildbot.plugins import util
from buildbot.reporters.gitlab import GitLabStatusPush
from buildbot.reporters.generators.build import BuildStatusGenerator,
BuildStartEndStatusGenerator
@@ -42,9 +44,24 @@ irc_reporter = irc.IRCStatusPush("irc.libera.chat",
"gci_test",
noticeOnChannel=True
)
#gitlab
+@util.renderer
+@defer.inlineCallbacks
+def getGitlabContext(props):
+context = None
+gentooci = props.master.namedServices['services'].namedServices['gentooci']
+if props.getProperty('buildername') == 'update_v_data':
+context = 'Update database and check ebuild'
+if props.getProperty('buildername') == 'run_build_request':
+project_data = yield
gentooci.db.projects.getProjectByUuid(props.getProperty('project_uuid'))
+context = f"Project/{project_data['description']}/Build"
+if props.getProperty('buildername') == 'parse_build_log':
+project_data = yield
gentooci.db.projects.getProjectByUuid(props.getProperty('project_build_data')['project_uuid'])
+context = f"Project/{project_data['description']}/Check Log"
+return context
def gitlabGenerators():
builders = [
-#'run_build_request',
+'update_v_data',
+'run_build_request',
'parse_build_log'
]
return [
@@ -53,7 +70,7 @@ def gitlabGenerators():
)
]
gitlab_gentoo_org = GitLabStatusPush(token=util.Secret("gitlabToken"),
-#context= util.Interpolate('Buildbot
%(prop:buildername)s'),
+context= getGitlabContext,
baseURL='https://gitlab.gentoo.org',
generators=gitlabGenerators(),
#debug=True,
[gentoo-commits] proj/tinderbox-cluster:master commit in: patches/, buildbot_gentoo_ci/steps/
commit: 3ce9ab6c909c80cef867ad0cedb7cda5b87fb111
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Nov 29 21:44:55 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Nov 29 21:44:55 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3ce9ab6c
Clean build and changes in db
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/clean.py | 70 ++-
patches/bb-clean-db.patch | 179 ++
patches/bb-deleteLogChunks.patch | 83 --
3 files changed, 228 insertions(+), 104 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/clean.py
b/buildbot_gentoo_ci/steps/clean.py
index 73733f1..e4d8573 100644
--- a/buildbot_gentoo_ci/steps/clean.py
+++ b/buildbot_gentoo_ci/steps/clean.py
@@ -31,6 +31,7 @@ class SetupPropertys(BuildStep):
days = 30
days_in_s = 86400 * days
now = int(self.master.reactor.seconds())
+bb_build_ids = []
for ebuild_data in deleted_ebuilds_data:
no_bug = True
# check if older the days
@@ -41,13 +42,20 @@ class SetupPropertys(BuildStep):
print(build_data_list)
for build_data in build_data_list:
if build_data['bug_id'] == 0:
-# get buildsets
print(f"Build : {build_data['id']} will be removed")
-yield
self.build.addStepsAfterCurrentStep([RemoveBuildFromDb(build_data)])
+if build_data['buildbot_build_id'] != 0:
+
bb_build_ids.append(build_data['buildbot_build_id'])
+# make a list with build_ids from parent_buildid
+# get parent_buildid from buildsets
+# bb_buildset = yield
self.master.db.buildsets.getBuildset(build_data['buildbot_build_id'])
+yield
self.gentooci.db.builds.removeBuild(build_data['id'])
else:
no_bug = False
if no_bug:
-yield self.build.
addStepsAfterLastStep([RemoveEbuildFromDb(ebuild_data)])
+yield
self.build.addStepsAfterCurrentStep([RemoveEbuildFromDb(ebuild_data['uuid'])])
+yield
self.build.addStepsAfterCurrentStep([RemoveChangesFromDb(ebuild_data['commit_id'])])
+for bb_build_id in bb_build_ids:
+yield
self.build.addStepsAfterCurrentStep([RemoveBuildFromDb(bb_build_id)])
return SUCCESS
class RemoveEbuildFromDb(BuildStep):
@@ -57,17 +65,34 @@ class RemoveEbuildFromDb(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, ebuild_data, **kwargs):
-self.ebuild_data = ebuild_data
+def __init__(self, uuid, **kwargs):
+self.uuid = uuid
super().__init__(**kwargs)
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
-yield
self.gentooci.db.versions.removeVersionKeyword(self.ebuild_data['uuid'])
-yield
self.gentooci.db.versions.removeVersionMetadata(self.ebuild_data['uuid'])
-yield self.gentooci.db.versions.removeVersion(self.ebuild_data['uuid'])
-self.descriptionDone = f"Ebuild : {self.ebuild_data['uuid']} will be
removed"
+yield self.gentooci.db.versions.removeVersionKeyword(self.uuid)
+yield self.gentooci.db.versions.removeVersionMetadata(self.uuid)
+yield self.gentooci.db.versions.removeVersion(self.uuid)
+self.descriptionDone = f"Ebuild : {self.uuid} will be removed"
+return SUCCESS
+
+class RemoveChangesFromDb(BuildStep):
+name = 'Clean Changes from db'
+description = 'Running'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, commit_id, **kwargs):
+self.commit_id = commit_id
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+yield self.master.db.changes.pruneChangesId(self.commit_id)
+self.descriptionDone = f"Change : {self.commit_id} will be removed"
return SUCCESS
class RemoveBuildFromDb(BuildStep):
@@ -77,21 +102,24 @@ class RemoveBuildFromDb(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, build_data, **kwargs):
-self.build_data = build_data
+def __init__(self, build_id, **kwargs):
+self.build_id = build_id
super().__init__(**kwargs)
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
-if self.build_data['buildbot_build_id'] != 0:
-self.descriptionDone = f"BuildBot Build :
{self.build_data['buildbot_build_id']} will be removed"
-bb_build_data = yield
self.master.db.builds.g
[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/steps/
commit: 684420873cc8bd4319c4379a78a76be25abe1e15
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Oct 28 07:03:48 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Oct 28 07:03:48 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=68442087
Match min words
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 34 --
gentooci.cfg | 3 ++-
2 files changed, 18 insertions(+), 19 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 4d958b7..6a097d4 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -16,6 +16,8 @@ from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
from buildbot.plugins import util
+from buildbot_gentoo_ci.utils.regex import stripQuotesAndMore, finishTitle
+
from bugz.cli import check_bugz_token, login, list_bugs
from bugz.cli_argparser import make_arg_parser
from bugz.configfile import load_config
@@ -120,28 +122,23 @@ class GetBugs(BuildStep):
yield log.addStdout(f"Bug: {str(bug['id'])} Summary:
{bug['summary']}\n")
# we splite the lines to lists and try to match the words
matches = 0
-match_search_text =
list(self.getProperty('error_dict')['title_issue'].split())
-match_bug_text = list(bug['summary'].split())
-#FIXME: add check for cp
-for match_word in match_search_text:
-if match_word in match_bug_text:
-matches = matches + 1
-# try to match the nice words
-matches_nice = 0
-match_search_text_nice =
list(self.getProperty('error_dict')['title_issue_nice'].split())
-#FIXME: add check for cp
-for match_word in match_search_text_nice:
-if match_word in match_bug_text:
-matches_nice = matches_nice + 1
-print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
{str(matches_nice)} Summary: {bug['summary']}")
-#FIXME: set it in bug_config
-if matches >= 6 or matches_nice >= 6:
+bug_words = list(bug['summary'].split())
+print(bug_words)
+print(self.getProperty('error_dict')['words'])
+for word in bug_words:
+# clean the word
+word = finishTitle(stripQuotesAndMore(word))
+print(word)
+for match_word in self.getProperty('error_dict')['words']:
+if match_word in word:
+matches = matches + 1
+print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
Summary: {bug['summary']}")
+if matches >= len(self.getProperty('error_dict')['title_nice']) -
self.bug_config['min_match_word']:
match = {}
match['match'] = True
match['id'] = bug['id']
match['summary'] = bug['summary']
yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue']}\n")
-yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue_nice']}\n")
if match:
yield log.addStdout('Match bug: YES\n')
yield log.addStdout(f"Bug: {str(match['id'])} Summary:
{match['summary']}\n")
@@ -152,7 +149,8 @@ class GetBugs(BuildStep):
@defer.inlineCallbacks
def run(self):
-# self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+self.bug_config = self.gentooci.config.project['bug_config']
cpv = self.getProperty('error_dict')['cpv']
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
diff --git a/gentooci.cfg b/gentooci.cfg
index 474da76..4aecf35 100644
--- a/gentooci.cfg
+++ b/gentooci.cfg
@@ -49,9 +49,10 @@ c['worker_config'] = {
# This specifies bug configs
c['bug_config'] = {
'enable' : True,
-'aout_assignee' : False,
+'auto_assignee' : False,
'user' : 'larrythe...@cow.mu',
'extra_summery' : '',
+'min_match_word' : 2,
'buildbot_comment' : [
'This was build on a tinderbox that use Buildbot API[1].',
'We use docker images to build on.',
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/utils/
commit: d35e389caa0fe40ffaf6ec8e6e460148716ed092
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Sep 21 22:22:00 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Sep 21 22:22:00 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d35e389c
Move nice regex's to a file
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 38 +++---
buildbot_gentoo_ci/utils/regex.py | 22 ++
2 files changed, 45 insertions(+), 15 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index d71eb99..85017a5 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -25,6 +25,7 @@ from buildbot.plugins import util
#from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
from buildbot_gentoo_ci.steps import bugs
+from buildbot_gentoo_ci.utils.regex import stripQuotesAndMore, finishTitle
def PersOutputOfLogParser(rc, stdout, stderr):
build_summery_output = {}
@@ -185,17 +186,23 @@ class MakeIssue(BuildStep):
super().__init__(**kwargs)
def getNiceErrorLine(self, full_line):
-# strip away hex addresses, loong path names, line and time numbers
and other stuff
-# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L469
-# FIXME: Add the needed line when needed
-new_line = []
-for line in full_line.split(' '):
-# Shorten the path
-if line.startswith('/usr/') or line.startswith('/var/') or
line.startswith('../'):
-split_path_line = os.path.split(line)
-line = line.replace(split_path_line[0], '...')
-new_line.append(line)
-return ' '.join(new_line)
+new_words = []
+for word in full_line.split(' '):
+new_words.append(finishTitle(stripQuotesAndMore(word)))
+return ' '.join(new_words)
+
+def BuildWordList(self):
+word_list = []
+word_list.append(self.error_dict['cpv'])
+c = catpkgsplit(self.error_dict['cpv'])[0]
+p = catpkgsplit(self.error_dict['cpv'])[1]
+cp = c + '/' + p
+word_list.append(cp)
+for word in self.error_dict['title_phase'].split(' '):
+word_list.append(word)
+for word in self.error_dict['title_issue_nice'].split(' '):
+word_list.append(word)
+return word_list
def ClassifyIssue(self):
# get the title for the issue
@@ -217,10 +224,11 @@ class MakeIssue(BuildStep):
self.error_dict['title_issue'] = 'title_issue : None'
self.error_dict['title_issue_nice'] = 'title_issue_nice : None'
self.error_dict['title_found'] = False
-self.error_dict['title_phase'] = 'fails to '+ self.error_dict['phase']
+ ':'
+self.error_dict['title_phase'] = 'fails to '+ self.error_dict['phase']
#set the error title
-self.error_dict['title'] = ' '.join([self.error_dict['title_phase'],
self.error_dict['title_issue']])
-self.error_dict['title_nice'] = '
'.join([self.error_dict['title_phase'], self.error_dict['title_issue_nice']])
+self.error_dict['title'] = ' '.join([self.error_dict['title_phase'] +
':', self.error_dict['title_issue']])
+self.error_dict['title_nice'] = '
'.join([self.error_dict['title_phase'] + ':',
self.error_dict['title_issue_nice']])
+self.error_dict['words'] = self.BuildWordList()
@defer.inlineCallbacks
def run(self):
@@ -247,7 +255,7 @@ class MakeIssue(BuildStep):
#FIXME: write summary_log_list to a file
# add issue/bug/pr report
if error:
-yield self.ClassifyIssue()
+self.ClassifyIssue()
print(self.error_dict)
self.setProperty("status", 'failed', 'status')
self.setProperty("error_dict", self.error_dict, 'error_dict')
diff --git a/buildbot_gentoo_ci/utils/regex.py
b/buildbot_gentoo_ci/utils/regex.py
new file mode 100644
index 000..29372e0
--- /dev/null
+++ b/buildbot_gentoo_ci/utils/regex.py
@@ -0,0 +1,22 @@
+# Copyright 2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import re
+
+# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L12
+def stripQuotesAndMore(word):
+word = re.sub(r"b'", "", word)
+word = re.sub(r"'", "", word)
+word = re.sub(r'`', '', word)
+word = re.sub(r'"', '', word)
+word = re.sub(r'\\', '', word)
+return word
+
+# strip away hex addresses, loong path names, line and time numbers and other
stuff
+# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L469
+# FIXME: Add the needed line when needed
+def finishTitle(word):
+if word.startswith('/'):
+word = word.split('/')[-1]
+word = re.sub(":\d+:\d+:", "", word)
+return word
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: db8e966eef3ede0212dcf85d6d5b82c28d3dfcd5
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 26 09:21:16 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 26 09:21:16 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=db8e966e
Check issue titel nice
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 13 +++--
buildbot_gentoo_ci/steps/logs.py | 31 +++
2 files changed, 26 insertions(+), 18 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index ad87be9..4d958b7 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -126,13 +126,22 @@ class GetBugs(BuildStep):
for match_word in match_search_text:
if match_word in match_bug_text:
matches = matches + 1
-print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
Summary: {bug['summary']}")
-if matches >= 5:
+# try to match the nice words
+matches_nice = 0
+match_search_text_nice =
list(self.getProperty('error_dict')['title_issue_nice'].split())
+#FIXME: add check for cp
+for match_word in match_search_text_nice:
+if match_word in match_bug_text:
+matches_nice = matches_nice + 1
+print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
{str(matches_nice)} Summary: {bug['summary']}")
+#FIXME: set it in bug_config
+if matches >= 6 or matches_nice >= 6:
match = {}
match['match'] = True
match['id'] = bug['id']
match['summary'] = bug['summary']
yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue']}\n")
+yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue_nice']}\n")
if match:
yield log.addStdout('Match bug: YES\n')
yield log.addStdout(f"Bug: {str(match['id'])} Summary:
{match['summary']}\n")
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 47112e0..d71eb99 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -184,18 +184,18 @@ class MakeIssue(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
-def getNiceErrorLine(self, line):
+def getNiceErrorLine(self, full_line):
# strip away hex addresses, loong path names, line and time numbers
and other stuff
-# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L467
+# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L469
# FIXME: Add the needed line when needed
-if re.search(': line', line):
-line = re.sub(r"\d", "", line)
-# Shorten the path
-if line.startswith('/usr/'):
-line = line.replace(os.path.split(line.split(' ', 1)[0])[0],
'/...')
-if re.search(': \d:\d: ', line):
-line = re.sub(r":\d:\d: ", ": ", line)
-return line
+new_line = []
+for line in full_line.split(' '):
+# Shorten the path
+if line.startswith('/usr/') or line.startswith('/var/') or
line.startswith('../'):
+split_path_line = os.path.split(line)
+line = line.replace(split_path_line[0], '...')
+new_line.append(line)
+return ' '.join(new_line)
def ClassifyIssue(self):
# get the title for the issue
@@ -215,11 +215,12 @@ class MakeIssue(BuildStep):
self.error_dict['title_found'] = True
else:
self.error_dict['title_issue'] = 'title_issue : None'
-self.error_dict['title_nice'] = 'title_issue : None'
+self.error_dict['title_issue_nice'] = 'title_issue_nice : None'
self.error_dict['title_found'] = False
-self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase']
+self.error_dict['title_phase'] = 'fails to '+ self.error_dict['phase']
+ ':'
#set the error title
-self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' +
self.error_dict['title_issue']
+self.error_dict['title'] = ' '.join([self.error_dict['title_phase'],
self.error_dict['title_issue']])
+self.error_dict['title_nice'] = '
'.join([self.error_dict['title_phase'], self.error_dict['title_issue_nice']])
@defer.inlineCallbacks
def run(self):
@@ -249,8 +250,6 @@ class MakeIssue(BuildStep):
yield self.ClassifyIssue()
print(self.error_dict)
self.setProperty("status", 'failed', 'status')
-#FIXME: nice description for post bug
-self.error_dict['summary_log_nice'] = ''
self.setProperty("error_dict", self.error_dict, 'error_dic
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
commit: 73e8b3e7c00b1b2686ae0df6d406181b6c3808a7 Author: Magnus Granberg gentoo org> AuthorDate: Mon Jun 26 21:49:07 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Mon Jun 26 21:49:07 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=73e8b3e7 Add support to remove ebuilds in db Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/db/versions.py | 40 +++ 1 file changed, 40 insertions(+) diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py index 6b534f3..0aaac1b 100644 --- a/buildbot_gentoo_ci/db/versions.py +++ b/buildbot_gentoo_ci/db/versions.py @@ -131,6 +131,46 @@ class VersionsConnectorComponent(base.DBConnectorComponent): res = yield self.db.pool.do(thd) return res +@defer.inlineCallbacks +def getEbuildsByPackage(self, p_uuid, deleted=False): +def thd(conn): +tbl = self.db.model.versions +q = tbl.select() +q = q.where(tbl.c.deleted == deleted) +q = q.where(tbl.c.package_uuid == p_uuid) +return [self._row2dict(conn, row) +for row in conn.execute(q).fetchall()] +res = yield self.db.pool.do(thd) +return res + +@defer.inlineCallbacks +def removeVersion(self, uuid): +def thd(conn, no_recurse=False): +tbl = self.db.model.versions +q = tbl.delete() +q = q.where(tbl.c.uuid == uuid) +conn.execute(q) +yield self.db.pool.do(thd) + +@defer.inlineCallbacks +def removeVersionMetadata(self, version_uuid): +def thd(conn, no_recurse=False): +tbl = self.db.model.versions_metadata +q = tbl.delete() +q = q.where(tbl.c.version_uuid == version_uuid) +conn.execute(q) +yield self.db.pool.do(thd) + +@defer.inlineCallbacks +def removeVersionKeyword(self, version_uuid): +def thd(conn, no_recurse=False): +tbl = self.db.model.versions_keywords +q = tbl.delete() +q = q.where(tbl.c.version_uuid == version_uuid) +conn.execute(q) +yield self.db.pool.do(thd) + + def _row2dict(self, conn, row): return dict( uuid=row.uuid,
[gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
commit: aead3c221debd25eba336987e4ad2cda67e870a1 Author: Magnus Granberg gentoo org> AuthorDate: Mon Jun 26 21:44:46 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Mon Jun 26 21:44:46 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aead3c22 Add deleteLogChunks patch for bb Signed-off-by: Magnus Granberg gentoo.org> patches/bb-deleteLogChunks.patch | 83 1 file changed, 83 insertions(+) diff --git a/patches/bb-deleteLogChunks.patch b/patches/bb-deleteLogChunks.patch new file mode 100644 index 000..6de332e --- /dev/null +++ b/patches/bb-deleteLogChunks.patch @@ -0,0 +1,83 @@ +--- a/buildbot/db/logs.py 2022-04-02 11:10:34.892310594 +0200 b/buildbot/db/logs.py 2023-06-26 23:06:24.611959431 +0200 +@@ -410,3 +410,80 @@ + rv = dict(row) + rv['complete'] = bool(rv['complete']) + return rv ++ ++# returns a Deferred that returns a value ++def deleteLogChunks(self, buildid): ++model = self.db.model ++horizon_per_builder = False ++ ++def countLogchunks(conn): ++res = conn.execute(sa.select([sa.func.count(model.logchunks.c.logid)])) ++count = res.fetchone()[0] ++res.close() ++return count ++ ++# find the steps.id at the upper bound of steps ++def getStepidMax(conn, buildid): ++# N.B.: we utilize the fact that steps.id is auto-increment, thus steps.started_at ++# times are effectively sorted and we only need to find the steps.id at the upper ++# bound of steps to update. ++ ++# SELECT steps.id from steps WHERE steps.buildid = buildid ORDER BY ++# steps.id DESC LIMIT 1; ++res = conn.execute( ++sa.select([model.steps.c.id]) ++.where(model.steps.c.buildid == buildid) ++.order_by(model.steps.c.id.desc()) ++.limit(1) ++) ++res_list = res.fetchone() ++stepid_max = None ++if res_list: ++stepid_max = res_list[0] ++res.close() ++return stepid_max ++ ++# query all logs with type 'd' and delete their chunks. ++def deleteLogsWithTypeD(conn): ++if self.db._engine.dialect.name == 'sqlite': ++# sqlite does not support delete with a join, so for this case we use a subquery, ++# which is much slower ++q = sa.select([model.logs.c.id]) ++q = q.select_from(model.logs) ++q = q.where(model.logs.c.type == 'd') ++ ++# delete their logchunks ++q = model.logchunks.delete().where(model.logchunks.c.logid.in_(q)) ++else: ++q = model.logchunks.delete() ++q = q.where(model.logs.c.id == model.logchunks.c.logid) ++q = q.where(model.logs.c.type == 'd') ++ ++res = conn.execute(q) ++res.close() ++ ++def thddeleteLogs(conn): ++count_before = countLogchunks(conn) ++ ++# update log types that match buildid ++# we do it first to avoid having UI discrepancy ++ ++stepid_max = getStepidMax(conn, buildid) ++if stepid_max: ++# UPDATE logs SET logs.type = 'd' ++# WHERE logs.stepid <= stepid_max AND type != 'd'; ++res = conn.execute( ++model.logs.update() ++.where(sa.and_(model.logs.c.stepid <= stepid_max, ++ model.logs.c.type != 'd')) ++.values(type='d') ++) ++res.close() ++ ++deleteLogsWithTypeD(conn) ++ ++count_after = countLogchunks(conn) ++count = count_before - count_after ++ ++return count if count > 0 else 0 ++return self.db.pool.do(thddeleteLogs)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/
commit: f95dce7cfc33e1d5a0e606c5562cac59c2dc0fdd
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Jun 26 21:52:10 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Jun 26 21:52:10 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f95dce7c
Add run_clean_db_request to clean db
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/builders.py | 29 +
buildbot_gentoo_ci/config/buildfactorys.py | 9 +++
buildbot_gentoo_ci/config/schedulers.py| 3 +
buildbot_gentoo_ci/steps/clean.py | 97 ++
buildbot_gentoo_ci/steps/package.py| 45 ++
5 files changed, 183 insertions(+)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index b3ddb16..584dfdf 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -20,6 +20,24 @@ def getWorkers(worker_type, workers):
worker_list.append(worker['uuid'])
return worker_list
+# check if we have same package_data
+@defer.inlineCallbacks
+def checkPackageData(master, builder, req1, req2):
+canBeCollapsed = yield buildrequest.BuildRequest.canBeCollapsed(master,
req1, req2)
+selfBuildset , otherBuildset = yield defer.gatherResults([
+master.data.get(('buildsets', req1['buildsetid'])),
+master.data.get(('buildsets', req2['buildsetid']))
+])
+print(f"TestCollapsed")
+print(f"canBeCollapsed: {canBeCollapsed}")
+print(f"selfBuildset: {selfBuildset}")
+print(f"otherBuildset: {otherBuildset}")
+if canBeCollapsed and selfBuildset['parent_buildid'] != None and \
+otherBuildset['parent_buildid'] != None:
+return True
+else:
+return False
+
@defer.inlineCallbacks
def CanWorkerBuildProject(builder, wfb, request):
gentooci =
builder.master.namedServices['services'].namedServices['gentooci']
@@ -118,4 +136,15 @@ def gentoo_builders(worker_data):
factory=buildfactorys.run_build_images_request()
)
)
+# Use multiplay workers
+b.append(util.BuilderConfig(
+name='run_clean_db_request',
+workernames=getWorkersLocal('local', worker_data['local']),
+# look builder so we only do one time
+# look=
+# if we allready runnin with same package_data properties then
skip/collapse
+#collapseRequests=checkPackageData,
+factory=buildfactorys.run_clean_db_request()
+)
+)
return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index b47a209..6755308 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -14,6 +14,7 @@ from buildbot_gentoo_ci.steps import portage
from buildbot_gentoo_ci.steps import logs
from buildbot_gentoo_ci.steps import repos
from buildbot_gentoo_ci.steps import nodes
+from buildbot_gentoo_ci.steps import clean
def update_db_check():
f = util.BuildFactory()
@@ -66,6 +67,8 @@ def update_db_cpv():
f.addStep(package.TriggerCheckForV())
# update metadata if needed
f.addStep(package.CheckMetadataPackagePath())
+# clean package db if needed (clean up ebuilds in db)
+f.addStep(package.TriggerCleanPackageDb())
return f
def update_db_v():
@@ -201,3 +204,9 @@ def run_build_images_request():
# set the needed steps for making the image
f.addStep(nodes.SetupBuildTypeAndSteps())
return f
+
+def run_clean_db_request():
+f = util.BuildFactory()
+# set needed Propertys
+f.addStep(clean.SetupPropertys())
+return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index 7833233..c34d714 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -92,6 +92,8 @@ def gentoo_schedulers():
builderNames=["parse_build_log"])
run_build_images_request =
schedulers.Triggerable(name="run_build_images_request",
builderNames=["run_build_images_request"])
+run_clean_db_request = schedulers.Triggerable(name="run_clean_db_request",
+ builderNames=["run_clean_db_request"])
s = []
s.append(create_images)
s.append(scheduler_update_db)
@@ -101,4 +103,5 @@ def gentoo_schedulers():
s.append(build_request_data)
s.append(run_build_request)
s.append(parse_build_log)
+s.append(run_clean_db_request)
return s
diff --git a/buildbot_gentoo_ci/steps/clean.py
b/buildbot_gentoo_ci/steps/clean.py
new file mode 100644
index 000..73733f1
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/clean.py
@@ -0,0 +1,97 @@
+# Copyright 2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from twisted.internet import defer
+from twisted.python import log
+
+fro
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, sql/, buildbot_gentoo_ci/steps/
commit: 2639fe0cd81fc82c4f9717daf756013cea6a86ca
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Jun 26 21:38:57 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Jun 26 21:38:57 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2639fe0c
Add bug id to builds data in db
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/builds.py | 56 +---
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/steps/logs.py | 7 +
sql/gentoo_ci_schema.sql | 1 +
4 files changed, 61 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
index 1cf2439..21adcc0 100644
--- a/buildbot_gentoo_ci/db/builds.py
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -15,7 +15,7 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.*
# Modifyed by Gentoo Authors.
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
import uuid
import sqlalchemy as sa
@@ -44,7 +44,9 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
requested=project_build_data['requested'],
created_at=created_at,
buildbot_build_id=0,
- build_id=new_number))
+ build_id=new_number,
+ bug_id=0
+ ))
except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
id = None
new_number = None
@@ -57,7 +59,6 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
def setStatusBuilds(self, id, status):
updated_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
-
tbl = self.db.model.projects_builds
q = tbl.update()
q = q.where(tbl.c.id == id)
@@ -69,10 +70,57 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
def setBuildbotBuildIdBuilds(self, id, buildbot_build_id):
updated_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
-
tbl = self.db.model.projects_builds
q = tbl.update()
q = q.where(tbl.c.id == id)
conn.execute(q, updated_at=updated_at,
buildbot_build_id=buildbot_build_id)
yield self.db.pool.do(thd)
+
+@defer.inlineCallbacks
+def setBugIdBuilds(self, id, bug_id):
+updated_at = int(self.master.reactor.seconds())
+def thd(conn, no_recurse=False):
+tbl = self.db.model.projects_builds
+q = tbl.update()
+q = q.where(tbl.c.id == id)
+conn.execute(q, updated_at=updated_at, bug_id=bug_id)
+yield self.db.pool.do(thd)
+
+@defer.inlineCallbacks
+def getBuildsByVersionUuid(self, uuid):
+def thd(conn):
+tbl = self.db.model.projects_builds
+q = tbl.select()
+q = q.where(tbl.c.version_uuid == uuid)
+res = conn.execute(q)
+row = res.fetchone()
+return [self._row2dict(conn, row)
+for row in conn.execute(q).fetchall()]
+res = yield self.db.pool.do(thd)
+return res
+
+@defer.inlineCallbacks
+def removeBuild(self, id):
+def thd(conn, no_recurse=False):
+tbl = self.db.model.projects_builds
+q = tbl.delete()
+q = q.where(tbl.c.id == id)
+conn.execute(q)
+yield self.db.pool.do(thd)
+
+def _row2dict(self, conn, row):
+return dict(
+id=row.id,
+build_id=row.build_id,
+project_uuid=row.project_uuid,
+version_uuid=row.version_uuid,
+buildbot_build_id=row.buildbot_build_id,
+bug_id=row.bug_id,
+status=row.status,
+requested=row.requested,
+created_at=row.created_at,
+updated_at=row.updated_at,
+deleted=row.deleted,
+deleted_at=row.deleted_at
+)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index b80281e..3ae4f92 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -243,6 +243,7 @@ class Model(base.DBConnectorComponent):
sa.ForeignKey('versions.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('buildbot_build_id', sa.Integer),
+sa.Column('bug_id', sa.Integer, nullable=False, default=0),
sa.Column('status',
sa.Enum('failed','completed','in-progress','waiting', 'warning'),
nullable=False),
sa.Column('requested', sa.Boolean, default=False),
sa.Column('created_at', sa.In
[gentoo-commits] proj/tinderbox-cluster:master commit in: py/, buildbot_gentoo_ci/steps/
commit: b5356f7674336d08ecbbe6a27602d12bb2beea5e
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Jun 10 10:04:53 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Jun 10 10:04:53 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b5356f76
Compression logs with xz
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 33 ++---
py/log_parser.py | 11 +++
2 files changed, 29 insertions(+), 15 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index a5638b5..117fb70 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -74,7 +74,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
if line.startswith('>>>'):
if line.startswith('>>> Failed to'):
emerge_output['failed'] = line.split(' ')[4][:-1]
-if line.endswith('.log.gz') and emerge_output['failed']:
+if line.endswith('.log') and emerge_output['failed']:
log_path_list.append(line.split(' ')[2])
#FIXME: Handling of >>> output
pass
@@ -83,7 +83,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
if line.startswith('!!! existing preserved libs'):
pass
if line.startswith(' * '):
-if line.endswith('.log.gz'):
+if line.endswith('.log'):
log_path_list.append(line.split(' ')[4])
#FIXME: Handling of depclean output dict of packages that get removed
or saved
emerge_output['packages'] = package_dict
@@ -122,7 +122,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
change_use[cpv_split[0]] = change_use_list
emerge_output['change_use'] = change_use
if line.startswith(' * '):
-if line.endswith('.log.gz'):
+if line.endswith('.log'):
log_path = line.split(' ')[3]
if log_path not in inlog_path_list:
log_path_list.append(log_path)
@@ -681,13 +681,13 @@ class CheckBuildWorkDirs(BuildStep):
def run(self):
cpv = self.getProperty('faild_cpv')
cpv_build_dir = yield os.path.join('/', 'var', 'tmp', 'portage',
self.getProperty('cpv_build_dir'))
-compressed_log_file = cpv.replace('/', '_') + '.' +
str(self.getProperty("buildnumber")) + '.logs.tar.bz2'
+compressed_log_file = cpv.replace('/', '_') + '.' +
str(self.getProperty("buildnumber")) + '.logs.tar.xz'
masterdest_file = yield os.path.join(self.getProperty('masterdest'),
compressed_log_file)
# cpv_build_work_dir = yield os.path.join(cpv_build_dir, 'work')
if self.getProperty('build_workdir_find_output')['build_workdir_find']
!= []:
shell_commad_list = []
shell_commad_list.append('tar')
-shell_commad_list.append('-cjpf')
+shell_commad_list.append('-cJvf')
shell_commad_list.append(compressed_log_file)
for find_line in
sorted(self.getProperty('build_workdir_find_output')['build_workdir_find']):
print(find_line)
@@ -695,7 +695,7 @@ class CheckBuildWorkDirs(BuildStep):
shell_commad_list.append(filename)
self.aftersteps_list.append(
steps.ShellCommand(
-name = 'Tar logs',
+name = 'Tar/xz logs',
command = shell_commad_list,
workdir = cpv_build_dir
))
@@ -761,12 +761,23 @@ class CheckEmergeLogs(BuildStep):
@defer.inlineCallbacks
def getLogFile(self, cpv, log_dict):
-file = log_dict[cpv]['full_logname']
-destfile = yield os.path.join(self.getProperty('masterdest'), file)
-sourcefile = log_dict[cpv]['log_path']
+compressed_log_file = log_dict[cpv]['full_logname'] + '.xz'
+destfile = yield os.path.join(self.getProperty('masterdest'),
compressed_log_file)
+sourcefile = log_dict[cpv]['log_path'] + '.xz'
name = 'Upload build log'
-url = '/'.join([hosturl, self.getProperty('workername'),
str(self.getProperty("buildnumber")), file])
-urlText = file
+url = '/'.join([hosturl, self.getProperty('workername'),
str(self.getProperty("buildnumber")), compressed_log_file])
+urlText = compressed_log_file
+shell_commad_list = []
+shell_commad_list.append('xz')
+shell_commad_list.append('-zv')
+shell_commad_list.append(log_dict[cpv]['full_logname'])
+self.aftersteps_list.append(
+steps.ShellCommand(
+name = 'Compress build log with xz',
+descriptionDone = log_dict[cpv]['full_logname'],
+command = shell_commad_list,
+workdir = os.path.dir
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: f664663c8bebcd6700c90e2ca445d82bdf9b8432
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Jun 10 09:53:48 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Jun 10 09:53:48 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f664663c
Add support for Post, Modify and Attach bugs
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 178 +--
1 file changed, 173 insertions(+), 5 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 6c85aac..ad87be9 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -1,7 +1,12 @@
-# Copyright 2022 Gentoo Authors
+# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+# Origins: bugz.cli
+# Modifyed by Gentoo Authors.
+
import re
+import os
+import xmlrpc
from twisted.internet import defer
@@ -9,6 +14,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
+from buildbot.plugins import util
from bugz.cli import check_bugz_token, login, list_bugs
from bugz.cli_argparser import make_arg_parser
@@ -19,8 +25,6 @@ from bugz.log import log_error, log_info
from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
-# Origins: bugz.cli
-# Modifyed by Gentoo Authors.
# main
def main_bugz(args):
ArgParser = make_arg_parser()
@@ -69,6 +73,32 @@ def search_bugz(args):
list_bugs(result, settings)
return result
+# post
+def post_bugs(args, params):
+"""Post a new bug"""
+settings = main_bugz(args)
+result = settings.call_bz(settings.bz.Bug.create, params)
+log_info('Bug %d submitted' % result['id'])
+return result
+
+# modify
+def modify_bugs(args, params):
+"""Modify an existing bug (eg. adding a comment or changing resolution.)"""
+settings = main_bugz(args)
+if len(params) < 2:
+raise BugzError('No changes were specified')
+result = settings.call_bz(settings.bz.Bug.update, params)
+return result
+
+def attach_bugs(args, params):
+""" Attach a file to a bug given a filename. """
+settings = main_bugz(args)
+fd = open(params['filename'], 'rb')
+params['data'] = xmlrpc.client.Binary(fd.read())
+fd.close()
+result = settings.call_bz(settings.bz.Bug.add_attachment, params)
+return result
+
class GetBugs(BuildStep):
name = 'GetBugs'
@@ -107,8 +137,9 @@ class GetBugs(BuildStep):
yield log.addStdout('Match bug: YES\n')
yield log.addStdout(f"Bug: {str(match['id'])} Summary:
{match['summary']}\n")
self.setProperty("bgo", match, 'bgo')
-return
+return match
yield log.addStdout('Match bug: NO\n')
+return match
@defer.inlineCallbacks
def run(self):
@@ -128,5 +159,142 @@ class GetBugs(BuildStep):
print(args)
buglist = search_bugz(args)
print(buglist)
-self.find_match(buglist)
+match = self.find_match(buglist)
+#FIXME: set bug id on build in db
+return SUCCESS
+
+class Post(BuildStep):
+
+name = 'Post bug'
+description = 'Running'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, bug_args, bugs_params, **kwargs):
+self.bug_args = bug_args
+self.bugs_params = bugs_params
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+#self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+#bug_config = self.gentooci.config.project['bug_config']
+args = []
+args.append('-u')
+args.append(self.bug_args['user'])
+args.append('-p')
+args.append(self.bug_args['passwd'])
+args.append('post')
+params = {}
+params['product'] = "Gentoo Linux"
+params['component'] = "Current packages"
+params['version'] = "unspecified"
+params['op_sys'] = "Linux"
+params['platform'] = "All"
+params['priority'] = "Normal"
+params['description'] = self.bugs_params['description']
+params['summary'] = self.bugs_params['summary']
+params['assigned_to'] = self.bugs_params['assigned_to']
+#params['cc'] = settings.cc
+params['url'] = self.bugs_params['url']
+bug_info = yield post_bugs(args, params)
+match = {}
+match['match'] = True
+match['id'] = bug_info['id']
+match['summary'] = self.bugs_params['summary']
+self.setProperty("bgo", match, 'bgo')
+print(match)
+self.descriptionDone = f"Bug: {bug_info['id']} submitted"
+#FIXME: update build with bug id
+return SUCCESS
+
+class Modify(Bui
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/
commit: e75fa8c7184989508dc3c445cf8b32fd99634d99 Author: Magnus Granberg gentoo org> AuthorDate: Sat Jun 10 10:06:11 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Sat Jun 10 10:06:11 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e75fa8c7 Add SetupBugReportSteps for bugreporting Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/buildfactorys.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index 2ca2ca9..b47a209 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -188,10 +188,10 @@ def parse_build_log(): f.addStep(logs.setEmergeInfoLog()) # add package info to log and db f.addStep(logs.setPackageInfoLog()) +# make the bugreport +f.addStep(logs.SetupBugReportSteps()) # set BuildStatus f.addStep(logs.setBuildStatus()) -# setup things for the irc bot -#f.addStep(logs.SetIrcInfo()) return f def run_build_images_request():
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, /, buildbot_gentoo_ci/steps/
commit: c2cd0bb34537771d61dd8d7c577cecf25c446200
Author: Magnus Granberg gentoo org>
AuthorDate: Sat Jun 10 10:02:20 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sat Jun 10 10:02:20 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c2cd0bb3
Add support for SetupBugReportSteps and compression with xz
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/config.py | 3 +
buildbot_gentoo_ci/steps/logs.py| 141
gentooci.cfg| 20 +
3 files changed, 132 insertions(+), 32 deletions(-)
diff --git a/buildbot_gentoo_ci/config/config.py
b/buildbot_gentoo_ci/config/config.py
index e2f33be..696ab0f 100644
--- a/buildbot_gentoo_ci/config/config.py
+++ b/buildbot_gentoo_ci/config/config.py
@@ -93,6 +93,7 @@ class GentooCiConfig(util.ComparableMixin):
"db_url",
"project",
"worker_config",
+"bug_config",
"repository_basedir"
])
@@ -152,3 +153,5 @@ class GentooCiConfig(util.ComparableMixin):
self.project['repository_basedir'] = DEFAULT_REPOSITORY_BASEDIR
if 'worker_config' in config_dict:
self.project['worker_config'] = config_dict['worker_config']
+if 'bug_config' in config_dict:
+self.project['bug_config'] = config_dict['bug_config']
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index e4fc951..ee11166 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
@@ -14,6 +14,7 @@ from twisted.internet import defer
from twisted.python import log
from buildbot.process.buildstep import BuildStep
+from buildbot.process.properties import Properties
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
@@ -21,7 +22,7 @@ from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
from buildbot.plugins import util
-from buildbot_gentoo_ci.steps import minio
+#from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
from buildbot_gentoo_ci.steps import bugs
@@ -91,13 +92,15 @@ class SetupPropertys(BuildStep):
self.setProperty("default_project_data", default_project_data,
'default_project_data')
self.setProperty("version_data", version_data, 'version_data')
self.setProperty("status", 'completed', 'status')
-if self.getProperty('faild_cpv'):
+if isinstance(self.getProperty('faild_cpv'), str):
log_cpv = self.getProperty('faild_cpv')
else:
log_cpv = self.getProperty('cpv')
self.setProperty("log_cpv", log_cpv, 'log_cpv')
-self.setProperty("bgo", dict( match=False), 'bgo')
+self.setProperty("bgo", False, 'bgo')
self.descriptionDone = 'Runing log checker on ' + log_cpv
+logsdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildbot_build_id']))
+self.setProperty("logsdir", logsdir, 'logsdir')
return SUCCESS
class SetupParserBuildLoger(BuildStep):
@@ -115,9 +118,9 @@ class SetupParserBuildLoger(BuildStep):
@defer.inlineCallbacks
def run(self):
self.aftersteps_list = []
-workdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildbot_build_id']))
log_cpv =
self.getProperty('log_build_data')[self.getProperty('log_cpv')]
-mastersrc_log = yield os.path.join(workdir, log_cpv['full_logname'])
+build_log_file_compressed = log_cpv['full_logname'] + '.xz'
+mastersrc_log = yield os.path.join(self.getProperty('logsdir'),
build_log_file_compressed)
log_py = 'log_parser.py'
config_log_py = 'logparser.json'
mastersrc_py = yield os.path.join(self.master.basedir, log_py)
@@ -125,7 +128,7 @@ class SetupParserBuildLoger(BuildStep):
# Upload logfile to worker
self.aftersteps_list.append(steps.FileDownload(
mastersrc=mastersrc_log,
-
workerdest=log_cpv['full_logname']
+
workerdest=build_log_file_compressed
))
# Upload log parser py code
self.aftersteps_list.append(steps.FileDownload(
@@ -137,6 +140,16 @@ class SetupParserBuildLoger(BuildStep):
mastersrc=mastersrc_config,
[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/
commit: 72a6e1e8bfdb606a6b6acee0abef43fdc13d5820
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jun 7 21:33:45 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jun 7 21:33:45 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=72a6e1e8
Add support for xz compression
Signed-off-by: Magnus Granberg gentoo.org>
docker/GentooBuildbotWorkerBuild.Dockerfile | 5 -
docker/GentooBuildbotWorkerLog.Dockerfile | 2 +-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/docker/GentooBuildbotWorkerBuild.Dockerfile
b/docker/GentooBuildbotWorkerBuild.Dockerfile
index 23fc3f0..e700203 100644
--- a/docker/GentooBuildbotWorkerBuild.Dockerfile
+++ b/docker/GentooBuildbotWorkerBuild.Dockerfile
@@ -35,6 +35,8 @@ RUN echo -e "[binhost]\npriority = \nsync-uri =
${BINHOSTURL}/${PROJECTUUID}
&& echo 'USE="X elogind"' | >> /etc/portage/make.conf\
# && echo 'PYTHON_TARGETS="python3_10 python3_9 python3_8 python3_11"' | >>
/etc/portage/make.conf\
&& echo 'ACCEPT_LICENSE="*"'\
+ && echo 'PORTAGE_COMPRESS="xz"'\
+ && echo 'BINPKG_COMPRESS="xz"'\
#&& echo 'dev-vcs/git -webdev -gnome-keyring' | cat >>
/etc/portage/package.use/git\
#&& echo 'dev-util/buildbot-worker' | cat >>
/etc/portage/package.accept_keywords/buildbot\
#&& echo 'dev-libs/glib' | cat >> /etc/portage/package.mask/git\
@@ -62,7 +64,8 @@ RUN echo 'app-text/ansifilter' | cat >>
/var/lib/portage/world\
&& echo 'dev-lang/rust-bin' | cat >> /var/lib/portage/world\
&& echo 'app-admin/eclean-kernel' | cat >> /var/lib/portage/world\
&& echo 'app-portage/gentoolkit' | cat >> /var/lib/portage/world\
- && echo 'app-editors/nano' | cat >> /var/lib/portage/world
+ && echo 'app-editors/nano' | cat >> /var/lib/portage/world\
+ && echo 'app-arch/xz-utils' | cat >> /var/lib/portage/world
# Run emerge update and install buildbot-worker
# update perl
diff --git a/docker/GentooBuildbotWorkerLog.Dockerfile
b/docker/GentooBuildbotWorkerLog.Dockerfile
index 492be36..0164b94 100644
--- a/docker/GentooBuildbotWorkerLog.Dockerfile
+++ b/docker/GentooBuildbotWorkerLog.Dockerfile
@@ -18,7 +18,7 @@ RUN echo -e "[binhost]\npriority = \nsync-uri =
https://gentoo.osuosl.org/ex
&& echo 'FEATURES="${FEATURES} buildpkg"' | cat >> /etc/portage/make.conf\
&& echo 'MAKEOPTS="-j8"' | cat >> /etc/portage/make.conf\
&& echo 'dev-libs/glib' | cat >> /etc/portage/package.mask/git\
- && emerge -qv buildbot-worker sqlalchemy dev-python/psycopg:2 rust-bin
dev-vcs/git
+ && emerge -qv buildbot-worker sqlalchemy dev-python/psycopg:2 rust-bin
dev-vcs/git utils-xz
# We don't need the gentoo repo any longer
RUN rm -R /var/db/repos/gentoo
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/
commit: 1abf3371807d22c82014ee335286db137a373300 Author: Magnus Granberg gentoo org> AuthorDate: Mon Jun 5 08:15:54 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Mon Jun 5 08:15:54 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1abf3371 Add requires to build (prepare) Signed-off-by: Magnus Granberg gentoo.org> sql/search_pattern.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/search_pattern.sql b/sql/search_pattern.sql index 1969442..65695a4 100644 --- a/sql/search_pattern.sql +++ b/sql/search_pattern.sql @@ -542,6 +542,7 @@ INSERT INTO public.projects_pattern VALUES (621, 'e89c2c1a-46e0-4ded-81dd-c51afe INSERT INTO public.projects_pattern VALUES (622, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'convert:', 0, 0, 'prepare', 'error', 'search'); INSERT INTO public.projects_pattern VALUES (623, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', ' \* .* not found', 0, 0, 'prepare', 'error', 'search'); INSERT INTO public.projects_pattern VALUES (624, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', ' \* Failed running', 0, 0, 'prepare', 'error', 'search'); +INSERT INTO public.projects_pattern VALUES (625, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', ' \* .* requires .* to build', 0, 0, 'prepare', 'error', 'search'); -- File CATCH.pretend INSERT INTO public.projects_pattern VALUES (640, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'Please switch to a gcc version built with USE=', 0, 0, 'pretend', 'error', 'search');
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/
commit: 599646e376fc489f091b4f53ec7574ca61cec63b
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Jun 4 20:59:43 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Jun 4 20:59:43 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=599646e3
Disable distfiles on docker
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/workers.py | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/config/workers.py
b/buildbot_gentoo_ci/config/workers.py
index 83eeda2..5f3a5a6 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -9,6 +9,8 @@ from buildbot.plugins import worker, util
def log_docker_images(props):
return 'bb-worker-log' + ':latest'
+#NOTE: Don't set distfiles if you have dist clean. Can get missing files
+#FIXME: Set in config
@util.renderer
def docker_volumes(props):
volumes_list = []
@@ -16,7 +18,7 @@ def docker_volumes(props):
src_dir = '/srv/gentoo/portage/' + props.getProperty('project_uuid')
dest_dir = '/var/cache/portage'
#add distdir
-volumes_list.append(src_dir + '/distfiles' + ':' + dest_dir + '/distfiles')
+#volumes_list.append(src_dir + '/distfiles' + ':' + dest_dir +
'/distfiles')
#add bindir
volumes_list.append(src_dir + '/packages' + ':' + dest_dir + '/packages')
return volumes_list
@@ -33,6 +35,7 @@ def docker_volumes_repositorys(props):
volumes_list.append(':'.join([src_dir, dest_dir]))
return volumes_list
+#FIXME: Move workers to DbConfig
http://docs.buildbot.net/current/manual/configuration/dbconfig.html
@util.renderer
@defer.inlineCallbacks
def getDockerHost(props, docker_worker):
@@ -45,6 +48,7 @@ def getDockerHost(props, docker_worker):
print(node_data)
return node_data['docker_host_url']
+#FIXME: Move workers to DbConfig
http://docs.buildbot.net/current/manual/configuration/dbconfig.html
@util.renderer
@defer.inlineCallbacks
def GetBuildDockerImage(props, docker_worker):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 89be950457b1ffebe2ba2a27e0b6dc7f455e3115
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Jun 4 20:50:42 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Jun 4 20:50:42 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=89be9504
Add fetchonly step but not on yet
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 30 ++
1 file changed, 30 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 8692813..a5638b5 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -564,6 +564,34 @@ class RunEmerge(BuildStep):
))
aftersteps_list.append(CheckEmergeLogs('pre-build'))
+if self.step == 'fetchonly':
+if projects_emerge_options['oneshot']:
+shell_commad_list.append('-1')
+shell_commad_list.append('-f')
+#shell_commad_list.append('--quiet=n')
+shell_commad_list.append('=' + self.getProperty('cpv'))
+# we don't use the bin for the requsted cpv
+shell_commad_list.append('--usepkg-exclude')
+shell_commad_list.append(cp)
+# don't build bin for virtual and acct-*
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('virtual')
+shell_commad_list.append('--buildpkg-exclude')
+shell_commad_list.append('acct-*')
+aftersteps_list.append(
+steps.ShellCommand(
+warnOnWarnings = True,
+warnOnFailure = True,
+flunkOnFailure = False,
+flunkOnWarnings = False,
+name = self.stepname,
+command=shell_commad_list,
+#strip=True,
+#extract_fn=PersOutputOfEmerge,
+workdir='/',
+timeout=self.build_timeout
+))
+
if self.step == 'build':
if projects_emerge_options['oneshot']:
shell_commad_list.append('-1')
@@ -787,6 +815,7 @@ class CheckEmergeLogs(BuildStep):
self.addFileUploade(sourcefile, destfile, name, url, urlText)
# get elogs
self.getElogFiles(cpv)
+#FIXME: add etc/portage dir to upload
@defer.inlineCallbacks
def getBuildWorkDirs(self, cpv):
@@ -1223,6 +1252,7 @@ class RunBuild(BuildStep):
aftersteps_list = []
aftersteps_list.append(RunEmerge(step='pre-build'))
aftersteps_list.append(RunEmergeInfo())
+#aftersteps_list.append(RunEmerge(step='fetchonly'))
aftersteps_list.append(RunEmerge(step='build'))
aftersteps_list.append(RunEmerge(step='pre-depclean'))
aftersteps_list.append(RunEmerge(step='preserved-libs'))
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 33a7f172aa06d773f11d2916130a48d16760c7d1
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Jun 4 20:57:28 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Jun 4 20:57:28 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=33a7f172
Support splited lines in auxdb output
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/version.py | 53 +
1 file changed, 24 insertions(+), 29 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index c6f333e..98e5403 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -26,11 +26,9 @@ def PersOutputOfGetEapi(rc, stdout, stderr):
# split the lines
for line in stdout.split('\n'):
if line.startswith('EAPI'):
-print(line[-1])
m = _pms_eapi_re.match(line)
if m is not None:
eapi = m.group(2)
-print(eapi)
if eapi is None or not eapi_is_supported(eapi):
print('ERROR: invalid eapi or not found')
eapi = False
@@ -41,37 +39,34 @@ def PersOutputOfGetEapi(rc, stdout, stderr):
}
def PersOutputOfGetAuxdb(rc, stdout, stderr):
-metadata = None
NoSplit = ['DESCRIPTION']
ignore_list = ['SRC_URI']
-#make dict of the stout
-index = 1
-metadata_line_dict = {}
-for text_line in stdout.splitlines():
-metadata_line_dict[index] = text_line
-index = index + 1
+metadata = {}
+i = 0
if not stderr == '':
print('stderr')
-# should have 22 lines
-if len(auxdbkeys) != index -1:
-# number of lines is incorrect.
-print('ERROR: Number of lines is incorrect')
-print(metadata_line_dict)
-return {
-'auxdb' : metadata
-}
-# split all keys to list instead of speces
-metadata = {}
-i = 1
-for key in auxdbkeys:
-if metadata_line_dict[i][-1] == '=' or key in ignore_list:
-metadata[key] = False
-else:
-if ' ' in metadata_line_dict[i] and key not in NoSplit:
-metadata[key] = metadata_line_dict[i].replace(key + '=',
'').split(' ')
-else:
-metadata[key] = []
-metadata[key].append(metadata_line_dict[i].replace(key + '=',
''))
+# make a dict of metadata
+metadata_line_list = stdout.splitlines()
+index = len(metadata_line_list)
+for metadata_line in metadata_line_list:
+print(metadata_line)
+for key in auxdbkeys:
+if metadata_line.startswith(key) and key not in ignore_list:
+print(key)
+if metadata_line[-1] == '=':
+metadata[key] = False
+else:
+if ' ' in metadata_line and key not in NoSplit:
+if i+1 == index or
metadata_line_list[i+1].startswith(auxdbkeys):
+metadata[key] = metadata_line.replace(key + '=',
'').split(' ')
+else:
+# support one splited line
+print(metadata_line_list[i+1])
+metadata_line2 = metadata_line +
metadata_line_list[i+1]
+metadata[key] = metadata_line2.replace(key + '=',
'').split(' ')
+else:
+metadata[key] = []
+metadata[key].append(metadata_line.replace(key + '=',
''))
i = i + 1
return {
'auxdb' : metadata
[gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
commit: 79ced78fa0f1b8a74e5582884e650225c658645e
Author: Magnus Granberg gentoo org>
AuthorDate: Sun May 28 09:51:02 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun May 28 09:51:02 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=79ced78f
Update gitlab patch to bb3.8
Signed-off-by: Magnus Granberg gentoo.org>
patches/bb-gitlab.patch | 15 +--
1 file changed, 1 insertion(+), 14 deletions(-)
diff --git a/patches/bb-gitlab.patch b/patches/bb-gitlab.patch
index 8c466f8..251991a 100644
--- a/patches/bb-gitlab.patch
+++ b/patches/bb-gitlab.patch
@@ -27,20 +27,7 @@ index 5a403d5d0..a6fc698e2 100644
log.msg(
f'Could not send status "{state}" for '
f'{sourcestamp["repository"]} at {sha}: {message}')
-diff --git a/buildbot/secrets/providers/file.py
b/buildbot/secrets/providers/file.py
-index 11daf327a..25e010b38 100644
a/buildbot/secrets/providers/file.py
-+++ b/buildbot/secrets/providers/file.py
-@@ -55,7 +55,7 @@ class SecretInAFile(SecretProviderBase):
- secretvalue = source.read()
- if suffix:
- secretfile = secretfile[:-len(suffix)]
--if strip:
-+if strip and not "OPENSSH PRIVATE KEY" in secretvalue:
- secretvalue = secretvalue.rstrip("\r\n")
- secrets[secretfile] = secretvalue
- return secrets
-diff --git a/master/buildbot/www/hooks/gitlab.py
b/master/buildbot/www/hooks/gitlab.py
+diff --git a/buildbot/www/hooks/gitlab.py b/buildbot/www/hooks/gitlab.py
index 8f2f80a83..40de0273e 100644
--- a/buildbot/www/hooks/gitlab.py
+++ b/buildbot/www/hooks/gitlab.py
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/
commit: 8f5231276de8cc3ad5a225a67f1f558de97d2410 Author: Magnus Granberg gentoo org> AuthorDate: Sun May 28 09:46:22 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Sun May 28 09:46:22 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8f523127 Set build_wait_timeout to 600 for log worker Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/workers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py index 6f27690..83eeda2 100644 --- a/buildbot_gentoo_ci/config/workers.py +++ b/buildbot_gentoo_ci/config/workers.py @@ -94,6 +94,6 @@ def gentoo_workers(worker_data): hostconfig=docker_hostconfig, followStartupLogs=True, masterFQDN='192.168.1.5', -#build_wait_timeout=3600 +build_wait_timeout=600 )) return w
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
commit: 2dfba1de696c99c0c03a8c1c6abd878b60f3c7fc
Author: Magnus Granberg gentoo org>
AuthorDate: Sun May 28 09:48:40 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun May 28 09:48:40 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2dfba1de
Update db python code to bb3.8
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/connector.py | 35 +-
buildbot_gentoo_ci/db/model.py | 231 ++---
2 files changed, 141 insertions(+), 125 deletions(-)
diff --git a/buildbot_gentoo_ci/db/connector.py
b/buildbot_gentoo_ci/db/connector.py
index 7665f84..0cc7884 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -15,7 +15,7 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.connector.py
# Modifyed by Gentoo Authors.
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
import textwrap
@@ -70,6 +70,10 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.setName('db')
self.basedir = basedir
+# not configured yet - we don't build an engine until the first
+# reconfig
+self.configured_url = None
+
# set up components
self._engine = None # set up in reconfigService
self.pool = None # set up in reconfigService
@@ -88,12 +92,16 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.builds = builds.BuildsConnectorComponent(self)
self.workers = workers.WorkersConnectorComponent(self)
+self.cleanup_timer = internet.TimerService(self.CLEANUP_PERIOD,
+ self._doCleanup)
+self.cleanup_timer.clock = self.master.reactor
+yield self.cleanup_timer.setServiceParent(self)
+
@defer.inlineCallbacks
def setup(self, config, check_version=True, verbose=True):
-db_url = config.db['db_url']
+db_url = self.configured_url = config.db['db_url']
-log.msg("Setting up database with URL %r"
-% util.stripUrlPassword(db_url))
+log.msg(f"Setting up database with URL
{repr(util.stripUrlPassword(db_url))}")
# set up the engine and pool
self._engine = enginestrategy.create_engine(db_url,
@@ -113,3 +121,22 @@ class DBConnector(service.ReconfigurableServiceMixin,
for l in
upgrade_message.format(basedir=self.basedir).split('\n'):
log.msg(l)
raise exceptions.DatabaseNotReadyError()
+
+def reconfigServiceWithBuildbotConfig(self, new_config):
+# double-check -- the master ensures this in config checks
+assert self.configured_url == new_config.db['db_url']
+
+return super().reconfigServiceWithBuildbotConfig(new_config)
+
+def _doCleanup(self):
+"""
+Perform any periodic database cleanup tasks.
+@returns: Deferred
+"""
+# pass on this if we're not configured yet
+if not self.configured_url:
+return None
+
+d = self.changes.pruneChanges(self.master.config.changeHorizon)
+d.addErrback(log.err, 'while pruning changes')
+return d
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 7ffe0ca..b80281e 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -1,7 +1,7 @@
# This file has parts from Buildbot and is modifyed by Gentoo Authors.
-# Buildbot is free software: you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation, version 2.
+# Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
@@ -18,10 +18,10 @@
# Copyright 2023 Gentoo Authors
import uuid
-import migrate
-import migrate.versioning.repository
+
+import alembic
+import alembic.config
import sqlalchemy as sa
-from migrate import exceptions # pylint: disable=ungrouped-imports
from twisted.internet import defer
from twisted.python import log
@@ -29,15 +29,36 @@ from twisted.python import util
from buildbot.db import base
from buildbot.db.migrate_utils import test_unicode
+from buildbot.db.types.json import JsonObject
from buildbot.util import sautils
-try:
-from migrate.versioning.schema import ControlledSchema # pylint:
disable=ungrouped-imports
-except ImportError:
-ControlledSchema = None
+
+class UpgradeFromBefore0p9Error(Exception):
+
+def __init__(self):
+message = """You are trying to upgrade a buildbot 0.8.x master to
buildbot 0.9.x or newer.
+This is not supported. Please start from a clean database
+
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/
commit: 2967519fc01d5041a31e47e98c630f222f5a9cf4
Author: Magnus Granberg gentoo org>
AuthorDate: Sun May 28 09:44:59 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun May 28 09:44:59 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2967519f
Use full url for Bugs
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/reporters.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/config/reporters.py
b/buildbot_gentoo_ci/config/reporters.py
index bd55037..54472b0 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -12,7 +12,7 @@ irc_template = '''{% set resultsList = ["\x0303SUCCESS",
"\x0308WARNINGS", "\x03
{{ build['properties']['revision'][0]|truncate(10, True) }} {{ "\x0302" }}{{
build['properties']['owners'][0][0] }}{{ "\x03" }} \
{{ "\x0306" }}{{ build['properties']['event'][0] }}{{ "\x03" }} {{ projects
}}:{{ build['properties']['project_data'][0]['name'] }} \
{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }} \
-{% if build['properties']['bgo'][0]['match'] is true %}{{ "\x0311" }}Bugid:
{{build['properties']['bgo'][0]['id']}}{{ "\x03" }}{% endif %}\
+{% if build['properties']['bgo'][0]['match'] is true %}{{ "\x0311" }}Bug: {{
"\x03" }}https://bugs.gentoo.org/{{build['properties']['bgo'][0]['id']}}{%
endif %}\
'''
def ircGenerators():
[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/
commit: 0d402a82fd38e399a0b8185734cd796d0215b5fc
Author: Magnus Granberg gentoo org>
AuthorDate: Mon May 8 20:14:25 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon May 8 20:14:25 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0d402a82
Use user buildbot for log image
Signed-off-by: Magnus Granberg gentoo.org>
docker/GentooBuildbotWorkerLog.Dockerfile | 26 --
1 file changed, 16 insertions(+), 10 deletions(-)
diff --git a/docker/GentooBuildbotWorkerLog.Dockerfile
b/docker/GentooBuildbotWorkerLog.Dockerfile
index a7c7af1..492be36 100644
--- a/docker/GentooBuildbotWorkerLog.Dockerfile
+++ b/docker/GentooBuildbotWorkerLog.Dockerfile
@@ -1,3 +1,6 @@
+# This docker file is use for the log parser and ebuild parser image
+# Use stable arch and gentoo's experimental binhost
+
# name the portage image
FROM gentoo/portage:latest as portage
@@ -7,24 +10,27 @@ FROM gentoo/stage3:latest
# copy the entire portage volume in
COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo
-# Setup portage
-# emerge needed deps buildbot-worker, psycopg, git and sqlalchemy
-# get the needed buildbot-worker config
+# Setup portage and emerge needed deps buildbot-worker, psycopg, git and
sqlalchemy
RUN echo -e "[binhost]\npriority = \nsync-uri =
https://gentoo.osuosl.org/experimental/amd64/binpkg/default/linux/17.1/x86-64/\n";
| cat >> /etc/portage/binrepos.conf\
&& echo 'EMERGE_DEFAULT_OPTS="--binpkg-respect-use=n --usepkg=y --getbinpkg=y
--autounmask-write --autounmask-continue --autounmask-keep-keywords=y
--autounmask-use=y"' | cat >> /etc/portage/make.conf\
&& echo 'FEATURES="-ipc-sandbox -pid-sandbox -network-sandbox -usersandbox
-mount-sandbox sandbox"' | cat >> /etc/portage/make.conf\
&& echo 'FEATURES="${FEATURES} parallel-install parallel-fetch -merge-sync"'
| cat >> /etc/portage/make.conf\
&& echo 'FEATURES="${FEATURES} buildpkg"' | cat >> /etc/portage/make.conf\
&& echo 'MAKEOPTS="-j8"' | cat >> /etc/portage/make.conf\
- && echo 'dev-vcs/git -webdev -gnome-keyring' | cat >>
/etc/portage/package.use/git\
- && echo 'dev-util/buildbot-worker' | cat >>
/etc/portage/package.accept_keywords/buildbot\
&& echo 'dev-libs/glib' | cat >> /etc/portage/package.mask/git\
- && emerge -qv buildbot-worker sqlalchemy dev-python/psycopg rust-bin
dev-vcs/git
- #&& chown buildbot:buildbot /var/lib/buildbot_worker
+ && emerge -qv buildbot-worker sqlalchemy dev-python/psycopg:2 rust-bin
dev-vcs/git
+
+# We don't need the gentoo repo any longer
+RUN rm -R /var/db/repos/gentoo
+
+# run on buildbot user
+RUN chown buildbot:buildbot /var/lib/buildbot_worker
+USER buildbot
-#FIXME: run worker as buildbot (git fail)
-#USER buildbot
WORKDIR /var/lib/buildbot_worker
-RUN wget
https://raw.githubusercontent.com/buildbot/buildbot/master/worker/docker/buildbot.tac
+# Get the needed buildbot-worker config
+# Change umask so portage can read the ebuilds in the repos
+RUN wget
https://raw.githubusercontent.com/buildbot/buildbot/master/worker/docker/buildbot.tac\
+ && ( sed -i -e 's/umask = None/umask = 0o22/g' buildbot.tac 2>/dev/null ||
true )
ENTRYPOINT ["/usr/bin/buildbot-worker"]
CMD ["start", "--nodaemon"]
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 6f1931fa4e868471cda59e88a1b1a8863753f9db
Author: Magnus Granberg gentoo org>
AuthorDate: Mon May 8 20:18:11 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon May 8 20:18:11 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6f1931fa
Use deep on eclean and check string on faild_cpv
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 30 +++---
1 file changed, 19 insertions(+), 11 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 64a3600..8692813 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
@@ -492,23 +492,23 @@ class RunEmerge(BuildStep):
# Run eclean on Packages and Distfiles
#FIXME: cleantime set in config
-cleantime_package = '1w'
+cleantime_package = '2d'
cleantime_distfiles = '2d'
if self.step == 'eclean':
aftersteps_list.append(
steps.ShellCommand(
flunkOnFailure=False,
name = 'Run eclean Packages',
-command=['eclean-pkg', '-Ct', cleantime_package],
+command=['eclean-pkg', '-C','-d', '-t' +
cleantime_package],
+workdir='/'
+))
+aftersteps_list.append(
+steps.ShellCommand(
+flunkOnFailure=False,
+name = 'Run eclean Distfiles',
+command=['eclean-dist', '-C', '-d', '-t' +
cleantime_distfiles],
workdir='/'
))
-#aftersteps_list.append(
-#steps.ShellCommand(
-#flunkOnFailure=False,
-#name = 'Run eclean Distfiles',
-#command=['eclean-dist', '-Ct', cleantime_distfiles],
-#workdir='/'
-#))
# Fix/Check Packages file
aftersteps_list.append(
steps.ShellCommand(
@@ -946,6 +946,7 @@ class CheckEmergeLogs(BuildStep):
log_path = log_path,
full_logname = full_logname
)
+print('log_dict')
print(log_dict)
if self.step == 'extra-build':
#FIXME: Check if extra build did work
@@ -962,15 +963,19 @@ class CheckEmergeLogs(BuildStep):
retrays = 1
# Find log for cpv that was requested or did failed
if not log_dict == {}:
+print('log_dict not emty')
# requested cpv
cpv = self.getProperty('cpv')
faild_version_data = False
if cpv in log_dict or self.faild_cpv in log_dict:
+print(cpv)
+print(self.faild_cpv)
yield self.createDistDir()
if cpv in log_dict:
self.log_data[cpv] = log_dict[cpv]
yield self.getLogFile(cpv, log_dict)
-if self.faild_cpv:
+if isinstance(self.faild_cpv, str):
+print('faild_cpv is string')
c = yield catpkgsplit(self.faild_cpv)[0]
if c == 'dev-haskell':
rebuild = 'haskell'
@@ -984,10 +989,13 @@ class CheckEmergeLogs(BuildStep):
yield self.getLogFile(self.faild_cpv, log_dict)
faild_version_data = yield
self.getVersionData(self.faild_cpv)
self.setProperty('faild_cpv', self.faild_cpv,
'faild_cpv')
+print(faild_version_data)
+print('getEmergeFiles')
self.getEmergeFiles(self.faild_cpv)
self.getBuildWorkDirs(self.faild_cpv)
rebuild = False
else:
+print('getEmergeFiles')
self.getEmergeFiles(cpv)
if not rebuild:
self.aftersteps_list.append(steps.Trigger(
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/
commit: b09c6350d3c98b8569a194f229b515d0508e8c35
Author: Magnus Granberg gentoo org>
AuthorDate: Mon May 1 09:58:12 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon May 1 09:58:12 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b09c6350
Add support for shallow in repositorys
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/model.py | 3 ++-
buildbot_gentoo_ci/db/repositorys.py | 9 -
buildbot_gentoo_ci/steps/repos.py| 3 ++-
sql/gentoo_ci_schema.sql | 1 +
4 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 6e13ad7..7ffe0ca 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -15,7 +15,7 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.model.py
# Modifyed by Gentoo Authors.
-# Copyright 2020 Gentoo Authors
+# Copyright 2023 Gentoo Authors
import uuid
import migrate
@@ -75,6 +75,7 @@ class Model(base.DBConnectorComponent):
sa.Column('type', sa.Enum('git', 'gitlab'), nullable=False,
default='git'),
sa.Column('mode', sa.Enum('full', 'incremental'), nullable=False,
default='incremental'),
sa.Column('method', sa.Enum('clobber', 'fresh', 'clean', 'copy'),
nullable=False, default='fresh'),
+sa.Column('shallow', sa.Integer, nullable=False, default=0),
sa.Column('alwaysuselatest', sa.Boolean, default=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
diff --git a/buildbot_gentoo_ci/db/repositorys.py
b/buildbot_gentoo_ci/db/repositorys.py
index 4d5d048..8fee1d3 100644
--- a/buildbot_gentoo_ci/db/repositorys.py
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -15,7 +15,7 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.*
# Modifyed by Gentoo Authors.
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
import sqlalchemy as sa
@@ -101,6 +101,12 @@ class
RepositorysConnectorComponent(base.DBConnectorComponent):
else:
sshprivatekey = row.sshprivatekey
sshhostkey = row.sshhostkey
+if row.shallow == 0:
+shallow = False
+elif row.shallow == 1:
+shallow = True
+else:
+shallow = row.shallow
return dict(
uuid=row.uuid,
name=row.name,
@@ -110,6 +116,7 @@ class
RepositorysConnectorComponent(base.DBConnectorComponent):
branch=branch,
mode=row.mode,
method=row.method,
+shallow=shallow,
alwaysuselatest=row.alwaysuselatest,
auto=row.auto,
enabled=row.enabled,
diff --git a/buildbot_gentoo_ci/steps/repos.py
b/buildbot_gentoo_ci/steps/repos.py
index 04b25d7..6d37906 100644
--- a/buildbot_gentoo_ci/steps/repos.py
+++ b/buildbot_gentoo_ci/steps/repos.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
@@ -216,6 +216,7 @@ class UpdateRepos(BuildStep):
branch = branch,
mode=repository_data['mode'],
method=repository_data['method'],
+shallow=repository_data['shallow'],
submodules=True,
alwaysUseLatest=repository_data['alwaysuselatest'],
workdir=repository_path,
diff --git a/sql/gentoo_ci_schema.sql b/sql/gentoo_ci_schema.sql
index 7de44ba..b166150 100644
--- a/sql/gentoo_ci_schema.sql
+++ b/sql/gentoo_ci_schema.sql
@@ -815,6 +815,7 @@ CREATE TABLE public.repositorys (
branch character varying(255),
mode public.repositorys_mode,
method public.repositorys_method,
+shallow integer NOT NULL,
alwaysuselatest boolean,
merge boolean,
sshprivatekey character varying(50),
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/
commit: b69003d51f5ed92cc38128cdc91c47e22eaa37b1 Author: Magnus Granberg gentoo org> AuthorDate: Mon May 1 09:53:42 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Mon May 1 09:53:42 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b69003d5 Don't use volumes for log worker Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/workers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py index 7d13f46..6f27690 100644 --- a/buildbot_gentoo_ci/config/workers.py +++ b/buildbot_gentoo_ci/config/workers.py @@ -90,7 +90,7 @@ def gentoo_workers(worker_data): None, docker_host='tcp://192.168.1.12:2375', image=log_docker_images, -volumes=docker_volumes_repositorys, +#volumes=docker_volumes_repositorys, hostconfig=docker_hostconfig, followStartupLogs=True, masterFQDN='192.168.1.5',
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: fe31d987a48588a27ded9bb0c1f0b49c69f273a8
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Mar 10 22:35:27 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Mar 10 22:35:27 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=fe31d987
Add CheckEAPI
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/version.py | 32 +++-
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index 59d19dc..c6f333e 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -409,6 +409,35 @@ class SetupPropertys(BuildStep):
self.setProperty("version", self.version, 'version')
return SUCCESS
+class CheckEAPI(BuildStep):
+
+name = 'CheckEAPI'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+#@defer.inlineCallbacks
+def run(self):
+# Run head 30 if we did't find it on head 10
+if self.getProperty("eapi"):
+return SKIPPED
+self.build.addStepsAfterCurrentStep([
+steps.SetPropertyFromCommand(
+name = 'RunGetEAPI',
+haltOnFailure = True,
+flunkOnFailure = True,
+command=['head', '-n','30',
self.getProperty("ebuild_file")],
+strip=False,
+extract_fn=PersOutputOfGetEapi
+)
+])
+return SUCCESS
+
class SetupStepsForCheckV(BuildStep):
name = 'Setup steps for Checking V'
@@ -441,10 +470,11 @@ class SetupStepsForCheckV(BuildStep):
name =
'RunGetEAPI',
haltOnFailure =
True,
flunkOnFailure =
True,
-command=['head',
'-n', '15', self.getProperty("ebuild_file")],
+command=['head',
'-n', '10', self.getProperty("ebuild_file")],
strip=False,
extract_fn=PersOutputOfGetEapi
))
+addStepVData.append(CheckEAPI())
addStepVData.append(steps.SetPropertyFromCommand(
name =
'GetPythonVersion',
haltOnFailure =
True,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: f9eed766d660ae99972201950f311c4cefc59be8 Author: Magnus Granberg gentoo org> AuthorDate: Fri Mar 10 22:33:41 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Fri Mar 10 22:33:41 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f9eed766 Download docker images befor build Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/nodes.py | 19 +++ 1 file changed, 19 insertions(+) diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py index f92727b..7a01cbb 100644 --- a/buildbot_gentoo_ci/steps/nodes.py +++ b/buildbot_gentoo_ci/steps/nodes.py @@ -108,6 +108,25 @@ class SetupBuildStepsForDocker(BuildStep): self.buildargs['BINHOSTURL'] = node_data['bin_host_url'] # set bootstrap image self.buildargs['BOOTSTRAPTAG'] = image_data['bootstrap_tag'] +# get the latest portage and bootstrap_tag images +aftersteps_list.append(steps.ShellCommand( +flunkOnFailure=True, +name='Pull ' + image_data['bootstrap_tag'], +command=['docker', + 'image', + 'pull', + image_data['bootstrap_tag'] + ] +)) +aftersteps_list.append(steps.ShellCommand( +flunkOnFailure=True, +name='Pull gentoo/portage:latest', +command=['docker', + 'image', + 'pull', + 'gentoo/portage:latest' + ] +)) # build the gentoo docker buildbot-worker image aftersteps_list.append(steps.ShellCommand( flunkOnFailure=True,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
commit: 165def4478ff2558075c7dcabdff85b2be31277b Author: Magnus Granberg gentoo org> AuthorDate: Fri Mar 10 22:32:06 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Fri Mar 10 22:32:06 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=165def44 Fix 2 bugs in db for email Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/db/packages.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildbot_gentoo_ci/db/packages.py b/buildbot_gentoo_ci/db/packages.py index 79ae685..1f68637 100644 --- a/buildbot_gentoo_ci/db/packages.py +++ b/buildbot_gentoo_ci/db/packages.py @@ -210,7 +210,8 @@ class PackagesConnectorComponent(base.DBConnectorComponent): tbl = self.db.model.packages_emails conn.execute(tbl.delete( whereclause=((tbl.c.package_uuid == package_uuid -return self.db.pool.do(thd) +res = yield self.db.pool.do(thd) +return res def _row2dict_email(self, conn, row): return dict( @@ -222,6 +223,6 @@ class PackagesConnectorComponent(base.DBConnectorComponent): id=row.id, email_id=row.email_id, package_uuid=row.package_uuid, -type=row.type, +mail_type=row.mail_type, proxied=row.proxied, )
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 1624134df50b603340e7e02d839f9d19476b52e8 Author: Magnus Granberg gentoo org> AuthorDate: Fri Mar 10 01:45:07 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Fri Mar 10 01:45:07 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1624134d Change cleantime_package to 1w and disable eclean-dist Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/builders.py | 16 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index a3f82a3..64a3600 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -492,7 +492,7 @@ class RunEmerge(BuildStep): # Run eclean on Packages and Distfiles #FIXME: cleantime set in config -cleantime_package = '3w' +cleantime_package = '1w' cleantime_distfiles = '2d' if self.step == 'eclean': aftersteps_list.append( @@ -502,13 +502,13 @@ class RunEmerge(BuildStep): command=['eclean-pkg', '-Ct', cleantime_package], workdir='/' )) -aftersteps_list.append( -steps.ShellCommand( -flunkOnFailure=False, -name = 'Run eclean Distfiles', -command=['eclean-dist', '-Ct', cleantime_distfiles], -workdir='/' -)) +#aftersteps_list.append( +#steps.ShellCommand( +#flunkOnFailure=False, +#name = 'Run eclean Distfiles', +#command=['eclean-dist', '-Ct', cleantime_distfiles], +#workdir='/' +#)) # Fix/Check Packages file aftersteps_list.append( steps.ShellCommand(
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, ...
commit: 0b3facc196912a1665b951a570cbafe76038ba03
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Mar 10 01:47:15 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Mar 10 01:47:15 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0b3facc1
Add package metadata support (email)
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 3 +-
buildbot_gentoo_ci/db/model.py | 28
buildbot_gentoo_ci/db/packages.py | 147 -
buildbot_gentoo_ci/steps/package.py| 166 +++-
sql/gentoo_ci_schema.sql | 201 ++---
5 files changed, 521 insertions(+), 24 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index efccc93..2ca2ca9 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -58,13 +58,14 @@ def update_db_cpv():
f.addStep(category.CheckC())
# if package in db
# return package_data
-# add check package path step at end
# else
# add package to db step
# return package_data
f.addStep(package.CheckP())
# Trigger update_db_v
f.addStep(package.TriggerCheckForV())
+# update metadata if needed
+f.addStep(package.CheckMetadataPackagePath())
return f
def update_db_v():
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 4373ebb..6e13ad7 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -287,6 +287,34 @@ class Model(base.DBConnectorComponent):
sa.Column('deleted_at', sa.Integer, nullable=True),
)
+emails = sautils.Table(
+"emails", metadata,
+# unique uuid per keyword
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('email', sa.String(255), nullable=False),
+)
+
+packages_emails = sautils.Table(
+"packages_emails", metadata,
+# unique uuid per keyword
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('package_uuid', sa.String(36),
+ sa.ForeignKey('packages.uuid', ondelete='CASCADE')),
+sa.Column('email_id', sa.Integer,
+ sa.ForeignKey('emails.id', ondelete='CASCADE')),
+sa.Column('mail_type', sa.Enum('project', 'person', 'unknown',),
nullable=False, default='unknown'),
+sa.Column('proxied', sa.Enum('yes', 'no', 'proxy',), nullable=False,
default='no'),
+)
+
+packages_metadata = sautils.Table(
+"packages_metadata", metadata,
+# unique uuid per keyword
+sa.Column('id', sa.Integer, primary_key=True),
+sa.Column('package_uuid', sa.String(36),
+ sa.ForeignKey('packages.uuid', ondelete='CASCADE')),
+sa.Column('sha256', sa.String(255), nullable=False, default='0'),
+)
+
versions = sautils.Table(
"versions", metadata,
sa.Column('uuid', sa.String(36), primary_key=True,
diff --git a/buildbot_gentoo_ci/db/packages.py
b/buildbot_gentoo_ci/db/packages.py
index a22b9d8..79ae685 100644
--- a/buildbot_gentoo_ci/db/packages.py
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -65,7 +65,10 @@ class PackagesConnectorComponent(base.DBConnectorComponent):
r = conn.execute(q, dict(name=name,
repository_uuid=repository_uuid,
category_uuid=category_uuid))
-except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+except Exception as e:
+print(type(e))
+print(e.args)
+print(e)
uuid = None
else:
uuid = r.inserted_primary_key[0]
@@ -80,3 +83,145 @@ class PackagesConnectorComponent(base.DBConnectorComponent):
repository_uuid=row.repository_uuid,
category_uuid=row.category_uuid
)
+
+@defer.inlineCallbacks
+def getPackageMetadataByPackageUuid(self, package_uuid):
+def thd(conn):
+tbl = self.db.model.packages_metadata
+q = tbl.select()
+q = q.where(tbl.c.package_uuid == package_uuid)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict_metadata(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
+@defer.inlineCallbacks
+def addMetadata(self, package_uuid, sha256):
+def thd(conn, no_recurse=False):
+try:
+tbl = self.db.model.packages_metadata
+q = tbl.insert()
+r = conn.execute(q, dict(package_uuid=package_uuid,
+ sha256=sha256
+ ))
+excep
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: c71f1b00885a6bd3851182ebc27d5eea67351cbc
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Feb 28 21:41:07 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Feb 28 21:41:07 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c71f1b00
Add eclean to building steps
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 29 +
1 file changed, 29 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 1746122..a3f82a3 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -490,6 +490,34 @@ class RunEmerge(BuildStep):
))
aftersteps_list.append(CheckDepcleanLogs('depclean'))
+# Run eclean on Packages and Distfiles
+#FIXME: cleantime set in config
+cleantime_package = '3w'
+cleantime_distfiles = '2d'
+if self.step == 'eclean':
+aftersteps_list.append(
+steps.ShellCommand(
+flunkOnFailure=False,
+name = 'Run eclean Packages',
+command=['eclean-pkg', '-Ct', cleantime_package],
+workdir='/'
+))
+aftersteps_list.append(
+steps.ShellCommand(
+flunkOnFailure=False,
+name = 'Run eclean Distfiles',
+command=['eclean-dist', '-Ct', cleantime_distfiles],
+workdir='/'
+))
+# Fix/Check Packages file
+aftersteps_list.append(
+steps.ShellCommand(
+flunkOnFailure=False,
+name='Fix/Check binary Package file',
+command=['emaint', 'binhost', '--fix'],
+workdir='/'
+))
+
if self.step == 'match':
shell_commad_list.append('-pO')
# don't use bin for match
@@ -1191,6 +1219,7 @@ class RunBuild(BuildStep):
aftersteps_list.append(RunEmerge(step='pre-depclean'))
aftersteps_list.append(RunEmerge(step='preserved-libs'))
aftersteps_list.append(RunEmerge(step='depclean'))
+aftersteps_list.append(RunEmerge(step='eclean'))
self.setProperty('depclean', False, 'depclean')
self.setProperty('preserved_libs', False, 'preserved-libs')
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: a2d7f047652600dc548a7401e3cc6b9b90cd7f1c
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Feb 28 21:40:12 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Feb 28 21:40:12 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=a2d7f047
Change head to 15 for EAPI search
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index 14d1d97..59d19dc 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -441,7 +441,7 @@ class SetupStepsForCheckV(BuildStep):
name =
'RunGetEAPI',
haltOnFailure =
True,
flunkOnFailure =
True,
-command=['head',
'-n', '10', self.getProperty("ebuild_file")],
+command=['head',
'-n', '15', self.getProperty("ebuild_file")],
strip=False,
extract_fn=PersOutputOfGetEapi
))
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/
commit: 7ca1b498984d6cb5c788cde2a68ba82d44e23965 Author: Magnus Granberg gentoo org> AuthorDate: Tue Feb 21 02:33:49 2023 + Commit: Magnus Granberg gentoo org> CommitDate: Tue Feb 21 02:33:49 2023 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7ca1b498 Add Could NOT find to search pattern db Signed-off-by: Magnus Granberg gentoo.org> sql/search_pattern.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/search_pattern.sql b/sql/search_pattern.sql index afa87d4..1969442 100644 --- a/sql/search_pattern.sql +++ b/sql/search_pattern.sql @@ -460,6 +460,7 @@ INSERT INTO public.projects_pattern VALUES (483, 'e89c2c1a-46e0-4ded-81dd-c51afe INSERT INTO public.projects_pattern VALUES (484, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', ', not found$', 0, 0, 'configure', 'info', 'search'); INSERT INTO public.projects_pattern VALUES (485, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'Could not find', 0, 0, 'configure', 'error', 'search'); INSERT INTO public.projects_pattern VALUES (486, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'No such file or directory', 0, 0, 'configure', 'error', 'search'); +INSERT INTO public.projects_pattern VALUES (487, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'Could NOT find', 0, 0, 'configure', 'error', 'search'); -- File CATCH.install INSERT INTO public.projects_pattern VALUES (500, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'Can''t locate Locale/gettext.pm in @INC', 0, 0, 'install', 'error', 'search');
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, py/
commit: 021a017b6e7ba89d28fac0e40db2ebb8baad9ce2
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Feb 21 01:39:54 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Feb 21 01:39:54 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=021a017b
Get all search pattern that match
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 4 +-
buildbot_gentoo_ci/steps/logs.py | 178 ---
py/log_parser.py | 53 +++-
3 files changed, 49 insertions(+), 186 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 801fc98..6c85aac 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -96,8 +96,8 @@ class GetBugs(BuildStep):
for match_word in match_search_text:
if match_word in match_bug_text:
matches = matches + 1
-if matches >= 10:
-print(f"Bug: {str(bug['id'])} Summary: {bug['summary']}")
+print(f"Bug: {str(bug['id'])} Matched words: {str(matches)}
Summary: {bug['summary']}")
+if matches >= 5:
match = {}
match['match'] = True
match['id'] = bug['id']
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 2a52308..e4fc951 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -19,6 +19,7 @@ from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
+from buildbot.plugins import util
from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
@@ -35,11 +36,15 @@ def PersOutputOfLogParser(rc, stdout, stderr):
for k, v in json.loads(line).items():
summary_log_dict[int(k)] = {
'text' : v['text'],
-'type' : v['type'],
-'status' : v['status'],
-'id' : v['id'],
-'search_pattern' : v['search_pattern']
+'pattern_infos' : [],
}
+for s in v['pattern_info']:
+summary_log_dict[int(k)]['pattern_infos'].append({
+'type' : s['type'],
+'status' :
s['status'],
+'id' : s['id'],
+'search_pattern' :
s['search_pattern'],
+})
build_summery_output['summary_log_dict'] = summary_log_dict
#FIXME: Handling of stderr output
return {
@@ -140,6 +145,8 @@ class SetupParserBuildLoger(BuildStep):
command.append(log_cpv['full_logname'])
command.append('-u')
command.append(self.getProperty('project_data')['uuid'])
+command.append('-d')
+command.append(util.Secret("log_parser_database"))
self.aftersteps_list.append(steps.SetPropertyFromCommand(
name =
'RunBuildLogParser',
haltOnFailure =
True,
@@ -152,159 +159,6 @@ class SetupParserBuildLoger(BuildStep):
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
-class ParserBuildLog(BuildStep):
-
-name = 'ParserBuildLog'
-description = 'Running'
-descriptionDone = 'Ran'
-descriptionSuffix = None
-haltOnFailure = True
-flunkOnFailure = True
-
-def __init__(self, **kwargs):
-self.logfile_text_dict = {}
-self.summery_dict = {}
-self.index = 1
-self.log_search_pattern_list = []
-self.max_text_lines = 0
-super().__init__(**kwargs)
-
-#FIXME: ansifilter
-def ansiFilter(self, text):
-return text
-
-@defer.inlineCallbacks
-def get_log_search_pattern(self):
-# get pattern from the projects
-# add that to log_search_pattern_list
-for project_pattern in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
-# check if the search pattern is vaild
-try:
-re.compile(project_pattern['search'])
-except re.error:
-print("Non valid regex pattern")
-print(project_pattern)
-else:
-self.log_search_pattern_list.append(project_pattern)
-# get the default project pattern
-# add if no
[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/, ...
commit: b38a60cfc5b5dca25a674ebe5887737525aa353f
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Feb 15 21:30:32 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Feb 15 21:30:32 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b38a60cf
Use command shell to build docker images
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/builders.py | 24 +++---
buildbot_gentoo_ci/config/buildfactorys.py | 6 +-
buildbot_gentoo_ci/config/schedulers.py | 22 ++---
buildbot_gentoo_ci/config/workers.py| 12 ++-
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/db/workers.py| 3 +-
buildbot_gentoo_ci/steps/nodes.py | 121
docker/GentooBuildbotWorkerBuild.Dockerfile | 31 ---
master.cfg | 14 +++-
sql/gentoo_ci_schema.sql| 14 +++-
10 files changed, 186 insertions(+), 62 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index edcd0c1..b3ddb16 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -13,7 +13,7 @@ def getWorkersLocal(worker_type, workers):
worker_list.append(worker['name'])
return worker_list
-def getWorkersDocker(worker_type, workers):
+def getWorkers(worker_type, workers):
worker_list = []
for worker in workers:
if worker['type'] == worker_type and worker['enable'] is True:
@@ -44,11 +44,11 @@ def CanWorkerUpdateV(builder, wfb, request):
print('Find a worker: NO')
return False
-def gentoo_builders(worker_data_local, worker_data_docker):
+def gentoo_builders(worker_data):
b = []
b.append(util.BuilderConfig(
name='update_db_check',
-workername=getWorkersLocal('local', worker_data_local)[0],
+workername=getWorkersLocal('local', worker_data['local'])[0],
workerbuilddir='builds',
collapseRequests=False,
factory=buildfactorys.update_db_check()
@@ -56,7 +56,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
)
b.append(util.BuilderConfig(
name='update_repo_check',
-workername=getWorkersLocal('local', worker_data_local)[1],
+workername=getWorkersLocal('local', worker_data['local'])[1],
workerbuilddir='builds',
collapseRequests=True,
factory=buildfactorys.update_repo_check()
@@ -67,7 +67,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
# (builders.UpdateRepos step)
b.append(util.BuilderConfig(
name='update_cpv_data',
-workernames=getWorkersDocker('log', worker_data_docker)[0],
+workernames=getWorkers('log', worker_data['docker'])[0],
workerbuilddir='builds',
collapseRequests=False,
factory=buildfactorys.update_db_cpv()
@@ -76,7 +76,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
# Use multiplay workers
b.append(util.BuilderConfig(
name='update_v_data',
-workername=getWorkersDocker('log', worker_data_docker)[0],
+workername=getWorkers('log', worker_data['docker'])[0],
workerbuilddir='builds',
collapseRequests=False,
canStartBuild=CanWorkerUpdateV,
@@ -86,7 +86,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
# Use multiplay workers
b.append(util.BuilderConfig(
name='build_request_data',
-workernames=getWorkersLocal('local', worker_data_local),
+workernames=getWorkersLocal('local', worker_data['local']),
collapseRequests=False,
factory=buildfactorys.build_request_check()
)
@@ -94,7 +94,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
# Use multiplay workers
b.append(util.BuilderConfig(
name='run_build_request',
-workernames=getWorkersDocker('build', worker_data_docker),
+workernames=getWorkers('build', worker_data['docker']),
canStartBuild=CanWorkerBuildProject,
collapseRequests=False,
factory=buildfactorys.run_build_request()
@@ -103,19 +103,19 @@ def gentoo_builders(worker_data_local,
worker_data_docker):
# Use multiplay workers
b.append(util.BuilderConfig(
name='parse_build_log',
-workernames=getWorkersDocker('log', worker_data_docker)[1:],
+workernames=getWorkers('log', worker_data['docker'])[1:],
collapseRequests=False,
factory=buildfactorys.parse_build_log()
)
)
# For node workers
b.append(util.BuilderConfig(
-name='run_build_stage4_request',
-workernames=getWorkersLocal('local', worker_data_local),
+name='run_build_images_request',
+workernames=getWorkers('node', worker_data['node']),
#FIXME: support more the one node
#canStart
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, sql/
commit: 102c51d88be07b23cd24d4cad6fad5e71edacb4c
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Feb 3 22:21:42 2023 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Feb 3 22:21:42 2023 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=102c51d8
Add support for binhost on docker image build
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/workers.py | 2 +-
buildbot_gentoo_ci/db/model.py | 3 ++-
buildbot_gentoo_ci/db/workers.py | 13 -
sql/gentoo_ci_schema.sql | 3 ++-
4 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/config/workers.py
b/buildbot_gentoo_ci/config/workers.py
index 3dcef1a..c29502e 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -43,7 +43,7 @@ def getDockerHost(props, docker_worker):
node_data = yield gentooci.db.workers.getNodeByUuid(node['node_uuid'])
print(worker_data)
print(node_data)
-return node_data['host_url']
+return node_data['docker_host_url']
@util.renderer
@defer.inlineCallbacks
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 87d7dbc..a4b5935 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -372,7 +372,8 @@ class Model(base.DBConnectorComponent):
sa.Column('uuid', sa.String(36), primary_key=True,
default=lambda: str(uuid.uuid4())),
sa.Column('name', sa.String(255), nullable=False),
-sa.Column('host_url', sa.String(255), nullable=False),
+sa.Column('docker_host_url', sa.String(255), nullable=False),
+sa.Column('bin_host_url', sa.String(255), nullable=False),
sa.Column('enable', sa.Boolean, default=False),
)
diff --git a/buildbot_gentoo_ci/db/workers.py b/buildbot_gentoo_ci/db/workers.py
index e82a6b2..d09d88c 100644
--- a/buildbot_gentoo_ci/db/workers.py
+++ b/buildbot_gentoo_ci/db/workers.py
@@ -130,6 +130,16 @@ class WorkersConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+@defer.inlineCallbacks
+def getAllNodes(self):
+def thd(conn):
+tbl = self.db.model.nodes
+q = tbl.select()
+return [self._row2dict_node(conn, row)
+for row in conn.execute(q).fetchall()]
+res = yield self.db.pool.do(thd)
+return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -164,7 +174,8 @@ class WorkersConnectorComponent(base.DBConnectorComponent):
return dict(
uuid=row.uuid,
name=row.name,
-host_url=row.host_url,
+docker_host_url=row.docker_host_url,
+bin_host_url=row.bin_host_url,
enable=row.enable
)
def _row2dict_node_worker(self, conn, row):
diff --git a/sql/gentoo_ci_schema.sql b/sql/gentoo_ci_schema.sql
index ff81661..5956664 100644
--- a/sql/gentoo_ci_schema.sql
+++ b/sql/gentoo_ci_schema.sql
@@ -274,7 +274,8 @@ ALTER TABLE public.migrate_version OWNER TO buildbot;
CREATE TABLE public.nodes (
name character varying,
-host_url character varying,
+docker_host_url character varying,
+bin_host_url character varying,
enable boolean,
uuid character varying(36) NOT NULL
);
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: ea8cb4a47f01964a09a7bd6f863d871a01c8b76d Author: Magnus Granberg gentoo org> AuthorDate: Wed Nov 9 21:05:15 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Wed Nov 9 21:05:15 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ea8cb4a4 Rename setBuildbotLog to Summarylog Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/buildfactorys.py | 2 +- buildbot_gentoo_ci/steps/logs.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index 7f2dff9..d07a143 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -180,7 +180,7 @@ def parse_build_log(): # set it SUCCESS/FAILURE/WARNINGS f.addStep(logs.MakeIssue()) # add sum log to buildbot log -f.addStep(logs.setBuildbotLog()) +f.addStep(logs.Summarylog()) # pers the emerge info f.addStep(logs.ReadEmergeInfoLog()) # add emerge info to log and db diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py index 67845e7..2a52308 100644 --- a/buildbot_gentoo_ci/steps/logs.py +++ b/buildbot_gentoo_ci/steps/logs.py @@ -404,9 +404,9 @@ class MakeIssue(BuildStep): yield self.build.addStepsAfterCurrentStep(self.aftersteps_list) return SUCCESS -class setBuildbotLog(BuildStep): +class Summarylog(BuildStep): -name = 'setBuildbotLog' +name = 'Summarylog' description = 'Running' descriptionDone = 'Ran' descriptionSuffix = None
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/, /, buildbot_gentoo_ci/config/
commit: 8e0586a1bd9bf20f83ff21eff6349d263b621cd3
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Oct 12 03:14:12 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Oct 12 03:14:12 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8e0586a1
Add config option for worker basedir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/config.py | 3 +++
buildbot_gentoo_ci/db/builds.py | 2 +-
buildbot_gentoo_ci/steps/builders.py | 4 ++--
buildbot_gentoo_ci/steps/logs.py | 2 +-
buildbot_gentoo_ci/steps/package.py | 11 +--
buildbot_gentoo_ci/steps/repos.py| 2 +-
buildbot_gentoo_ci/steps/version.py | 11 ---
gentooci.cfg | 9 -
8 files changed, 33 insertions(+), 11 deletions(-)
diff --git a/buildbot_gentoo_ci/config/config.py
b/buildbot_gentoo_ci/config/config.py
index faf2058..e2f33be 100644
--- a/buildbot_gentoo_ci/config/config.py
+++ b/buildbot_gentoo_ci/config/config.py
@@ -92,6 +92,7 @@ class GentooCiConfig(util.ComparableMixin):
_known_config_keys = set([
"db_url",
"project",
+"worker_config",
"repository_basedir"
])
@@ -149,3 +150,5 @@ class GentooCiConfig(util.ComparableMixin):
self.project['repository_basedir'] =
config_dict['repository_basedir']
else:
self.project['repository_basedir'] = DEFAULT_REPOSITORY_BASEDIR
+if 'worker_config' in config_dict:
+self.project['worker_config'] = config_dict['worker_config']
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
index 57302ef..1cf2439 100644
--- a/buildbot_gentoo_ci/db/builds.py
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -54,7 +54,7 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
return self.db.pool.do(thd)
@defer.inlineCallbacks
-def setStausBuilds(self, id, status):
+def setStatusBuilds(self, id, status):
updated_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index aed9edb..1746122 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -316,7 +316,7 @@ class SetupPropertys(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
-self.setProperty('portage_repos_path',
self.gentooci.config.project['project']['worker_portage_repos_path'],
'portage_repos_path')
+self.setProperty('portage_repos_path',
self.gentooci.config.project['worker_config']['portage_repos_path'],
'portage_repos_path')
self.setProperty('rootworkdir', False, 'rootworkdir')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
@@ -333,7 +333,7 @@ class SetupPropertys(BuildStep):
project_build_data = self.getProperty('project_build_data')
project_build_data['status'] = 'in-progress'
project_build_data['buildbot_build_id'] =
self.getProperty("buildnumber")
-yield self.gentooci.db.builds.setStausBuilds(
+yield self.gentooci.db.builds.setStatusBuilds(
project_build_data['id'],
project_build_data['status'])
yield self.gentooci.db.builds.setBuildbotBuildIdBuilds(
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index becf7d1..67845e7 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -596,7 +596,7 @@ class setBuildStatus(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
project_build_data = self.getProperty('project_build_data')
-yield self.gentooci.db.builds.setStausBuilds(
+yield self.gentooci.db.builds.setStatusBuilds(
project_build_data['id'],
self.getProperty('status')
)
diff --git a/buildbot_gentoo_ci/steps/package.py
b/buildbot_gentoo_ci/steps/package.py
index 81b400d..9fc59e8 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -29,8 +29,15 @@ class SetupPropertys(BuildStep):
#@defer.inlineCallbacks
def run(self):
-self.setProperty('portage_repos_path', '/repositorys',
'portage_repos_path')
-self.setProperty('rootworkdir', '/var/lib/buildbot_worker',
'rootworkdir')
+self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+if self.gentooci.config.project['worker_config']['basedir'
[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/
commit: 5feaf00fc4770fbf3088244faa129996d1b08b8c
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Oct 5 09:06:09 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Oct 5 09:06:09 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5feaf00f
Fix the Log and Build dockerfiles
Signed-off-by: Magnus Granberg gentoo.org>
docker/GentooBuildbotWorker.Dockerfile | 14
docker/GentooBuildbotWorkerBuild.Dockerfile| 79 ++
...ckerfile => GentooBuildbotWorkerLog.Dockerfile} | 0
3 files changed, 79 insertions(+), 14 deletions(-)
diff --git a/docker/GentooBuildbotWorker.Dockerfile
b/docker/GentooBuildbotWorker.Dockerfile
deleted file mode 100644
index db5e398..000
--- a/docker/GentooBuildbotWorker.Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-# This Dockerfile creates a gentoo buildbot worker container image
-# from a gentoo project stage4 docker image.
-
-ARG PROJECTUUID
-
-FROM stage4-${PROJECTUUID}
-RUN echo "Building Gentoo Buildbot worker Container image for ${PROJECTUUID}" \
- && ( sed -i -e 's/#rc_sys=""/rc_sys="docker"/g' etc/rc.conf 2>/dev/null ||
true ) \
- && echo 'UTC' > etc/timezone \
- && echo 'docker' >> /var/lib/buildbot_worker/info/host \
- && echo 'bb-worker-${PROJECTUUID}:latest' >>
/var/lib/buildbot_worker/info/host
-WORKDIR /var/lib/buildbot_worker
-ENTRYPOINT ["/usr/bin/buildbot-worker"]
-CMD ["start", "--nodaemon"]
diff --git a/docker/GentooBuildbotWorkerBuild.Dockerfile
b/docker/GentooBuildbotWorkerBuild.Dockerfile
new file mode 100644
index 000..bd357dc
--- /dev/null
+++ b/docker/GentooBuildbotWorkerBuild.Dockerfile
@@ -0,0 +1,79 @@
+# This Dockerfile creates a gentoo stage4 container image with buildbot worker
installed.
+
+ARG TAG=latest
+
+FROM gentoo/portage:latest as portage
+
+FROM gentoo/stage3:$TAG
+
+ARG KEYWORDS
+ARG PROJECTUUID
+ARG BINHOSTURL
+ARG ARCH
+ARG TAG=latest
+ARG NODE
+
+# copy the entire portage volume in
+COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo
+
+# Setup portage for git install
+RUN echo 'FEATURES="-ipc-sandbox -pid-sandbox -network-sandbox -usersandbox
-mount-sandbox sandbox"' | cat >> /etc/portage/make.conf\
+ && echo 'MAKEOPTS="-j8"' | cat >> /etc/portage/make.conf
+
+# Install git
+RUN emerge -qv dev-vcs/git
+
+# Setup portage for the rest of the build
+RUN echo -e "[binhost]\npriority = \nsync-uri =
${BINHOSTURL}/${PROJECTUUID}/packages/\n" | cat >> /etc/portage/binrepos.conf\
+ && echo 'EMERGE_DEFAULT_OPTS="--buildpkg=y --rebuild-if-new-rev=y
--rebuilt-binaries=y --usepkg=y --binpkg-respect-use=n --binpkg-changed-deps=y
--nospinner --color=n --ask=n"' | cat >> /etc/portage/make.conf\
+ && echo 'FEATURES="${FEATURES} parallel-fetch -merge-sync "' | cat >>
/etc/portage/make.conf\
+ && echo 'FEATURES="${FEATURES} getbinpkg"' | cat >> /etc/portage/make.conf\
+ && echo 'COMMON_FLAGS="${COMMON_FLAGS} -fno-diagnostics-color -march=native"'
| >> /etc/portage/make.conf\
+ && echo 'USE="X elogind"' | >> /etc/portage/make.conf\
+ && echo 'PYTHON_TARGETS="python3_10 python3_9 python3_8 python3_11"' | >>
/etc/portage/make.conf\
+ && echo 'ACCEPT_LICENSE="*"'\
+ && ( sed -i -e 's/#rc_sys=""/rc_sys="docker"/g' /etc/rc.conf 2>/dev/null ||
true ) \
+ && echo 'UTC' > /etc/timezone
+
+# Setup LANG
+RUN echo 'en_US.UTF-8 UTF-8' | cat >> /etc/locale.gen\
+ && echo 'en_US ISO-8859-1' | cat >> /etc/locale.gen\
+ && echo 'C.UTF8 UTF-8' | cat >> /etc/locale.gen\
+ && echo 'LANG="en_US.utf8"' | cat >> /etc/locale.conf\
+ && locale-gen
+
+# clone gentoo repo use github https for
+# gitlab.gentoo.org do timeout on https
+RUN rm -R /var/db/repos/gentoo\
+ && git clone --depth=1 https://github.com/gentoo/gentoo.git
/var/db/repos/gentoo
+
+# set accept_keywords
+RUN echo 'ACCEPT_KEYWORDS="'"${KEYWORDS}"'"' | cat >> /etc/portage/make.conf
+
+# Setup world file
+RUN echo 'app-text/ansifilter' | cat >> /var/lib/portage/world\
+ && echo 'dev-util/pkgcheck' | cat >> /var/lib/portage/world\
+ && echo 'dev-lang/rust-bin' | cat >> /var/lib/portage/world\
+ && echo 'app-admin/eclean-kernel' | cat >> /var/lib/portage/world\
+ && echo 'app-portage/gentoolkit' | cat >> /var/lib/portage/world\
+ && echo 'app-editors/nano' | cat >> /var/lib/portage/world
+
+# Run emerge update and install buildbot-worker
+RUN emerge -1qv perl\
+ && perl-cleaner --all\
+ && emerge -qv --update --deep --newuse @world\
+ && emerge -qv gentoo-kernel-bin\
+ && emerge -qv buildbot-worker
+
+# Rm the gentoo repo
+RUN rm -R /var/db/repos/gentoo
+
+# Setup the buildbot-worker
+WORKDIR /var/lib/buildbot_worker
+RUN wget
https://raw.githubusercontent.com/buildbot/buildbot/master/worker/docker/buildbot.tac\
+ && ( sed -i -e 's/umask = None/umask = 0o22/g' buildbot.tac 2>/dev/null ||
true ) \
+ && mkdir info\
+ && echo "${PROJECTUUID} ${ARCH} stage3:${TAG} ${NODE}" | cat > info/host
+ENTRYPOINT ["/usr/bin/buildbot-worker"]
+CMD ["start", "--nodaemon"]
+#CMD ["/bin/bash"]
d
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/
commit: 74dc875a027df21c3aa0504e4fc9526fb3c6944d Author: Magnus Granberg gentoo org> AuthorDate: Tue Oct 4 23:58:19 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Tue Oct 4 23:58:19 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=74dc875a Fix owner to buildbot Signed-off-by: Magnus Granberg gentoo.org> sql/gentoo_ci_schema.sql | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/gentoo_ci_schema.sql b/sql/gentoo_ci_schema.sql index fc10eff..ff81661 100644 --- a/sql/gentoo_ci_schema.sql +++ b/sql/gentoo_ci_schema.sql @@ -137,7 +137,7 @@ CREATE TYPE public.projects_status AS ENUM ( ALTER TYPE public.projects_status OWNER TO buildbot; -- --- Name: repositorys_method; Type: TYPE; Schema: public; Owner: postgres +-- Name: repositorys_method; Type: TYPE; Schema: public; Owner: buildbot -- CREATE TYPE public.repositorys_method AS ENUM ( @@ -148,10 +148,10 @@ CREATE TYPE public.repositorys_method AS ENUM ( ); -ALTER TYPE public.repositorys_method OWNER TO postgres; +ALTER TYPE public.repositorys_method OWNER TO buildbot; -- --- Name: repositorys_mode; Type: TYPE; Schema: public; Owner: postgres +-- Name: repositorys_mode; Type: TYPE; Schema: public; Owner: buildbot -- CREATE TYPE public.repositorys_mode AS ENUM ( @@ -160,7 +160,7 @@ CREATE TYPE public.repositorys_mode AS ENUM ( ); -ALTER TYPE public.repositorys_mode OWNER TO postgres; +ALTER TYPE public.repositorys_mode OWNER TO buildbot; -- -- Name: repositorys_type; Type: TYPE; Schema: public; Owner: buildbot
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
commit: 4827e44a0e0e6df8786bec353c7f01a39af4e089 Author: Magnus Granberg gentoo org> AuthorDate: Mon Oct 3 01:20:32 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Mon Oct 3 01:20:32 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4827e44a Fix bootstrap_tab in workers db Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/db/workers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_gentoo_ci/db/workers.py b/buildbot_gentoo_ci/db/workers.py index afc7815..e82a6b2 100644 --- a/buildbot_gentoo_ci/db/workers.py +++ b/buildbot_gentoo_ci/db/workers.py @@ -151,7 +151,7 @@ class WorkersConnectorComponent(base.DBConnectorComponent): name=row.name, tag=row.tag, dockerfile=row.dockerfile, -bootstrap_image_tag=row.bootstrap_image_tag +bootstrap_tag=row.bootstrap_tag ) def _row2dict_worker_config(self, conn, row): return dict(
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, sql/, /, buildbot_gentoo_ci/db/, patches/
commit: 2202c53f0179002adde3095609d89fcee394b614
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Oct 2 14:13:30 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Oct 2 14:13:30 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2202c53f
Add support for workes, nodes, images and flavors in db
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/builders.py | 41 +++--
buildbot_gentoo_ci/config/workers.py | 81 +
buildbot_gentoo_ci/db/connector.py| 2 +
buildbot_gentoo_ci/db/model.py| 57 ++-
buildbot_gentoo_ci/db/workers.py | 175 +++
master.cfg| 37 ++--
patches/bb-props_master_fix.patch | 15 ++
sql/gentoo_ci_schema.sql | 308 +++---
8 files changed, 467 insertions(+), 249 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index 82552fa..edcd0c1 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -1,11 +1,24 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2022 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from twisted.internet import defer
from buildbot.plugins import util
from buildbot_gentoo_ci.config import buildfactorys
-from buildbot_gentoo_ci.config.workers import gentoo_ci_workers
+
+def getWorkersLocal(worker_type, workers):
+worker_list = []
+for worker in workers:
+if worker['type'] == worker_type and worker['enable'] is True:
+worker_list.append(worker['name'])
+return worker_list
+
+def getWorkersDocker(worker_type, workers):
+worker_list = []
+for worker in workers:
+if worker['type'] == worker_type and worker['enable'] is True:
+worker_list.append(worker['uuid'])
+return worker_list
@defer.inlineCallbacks
def CanWorkerBuildProject(builder, wfb, request):
@@ -16,8 +29,9 @@ def CanWorkerBuildProject(builder, wfb, request):
print(wfb)
for worker in project_workers:
if wfb.worker.workername == worker['worker_uuid']:
+print('Find a worker: YES')
return True
-print('no worker')
+print('Find a worker: NO')
return False
# Use same worker as update_cpv_data was done by so we have same git commit
@@ -25,15 +39,16 @@ def CanWorkerUpdateV(builder, wfb, request):
print(request.properties['cp_worker'])
print(wfb)
if wfb.worker.workername == request.properties['cp_worker']:
+print('Find a worker: YES')
return True
+print('Find a worker: NO')
return False
-def gentoo_builders(worker_data):
+def gentoo_builders(worker_data_local, worker_data_docker):
b = []
-g_ci_w = gentoo_ci_workers(worker_data)
b.append(util.BuilderConfig(
name='update_db_check',
-workername=g_ci_w.getWorkersUuid('local')[0],
+workername=getWorkersLocal('local', worker_data_local)[0],
workerbuilddir='builds',
collapseRequests=False,
factory=buildfactorys.update_db_check()
@@ -41,7 +56,7 @@ def gentoo_builders(worker_data):
)
b.append(util.BuilderConfig(
name='update_repo_check',
-workername=g_ci_w.getWorkersUuid('local')[1],
+workername=getWorkersLocal('local', worker_data_local)[1],
workerbuilddir='builds',
collapseRequests=True,
factory=buildfactorys.update_repo_check()
@@ -52,7 +67,7 @@ def gentoo_builders(worker_data):
# (builders.UpdateRepos step)
b.append(util.BuilderConfig(
name='update_cpv_data',
-workernames=g_ci_w.getWorkersUuid('log')[0],
+workernames=getWorkersDocker('log', worker_data_docker)[0],
workerbuilddir='builds',
collapseRequests=False,
factory=buildfactorys.update_db_cpv()
@@ -61,7 +76,7 @@ def gentoo_builders(worker_data):
# Use multiplay workers
b.append(util.BuilderConfig(
name='update_v_data',
-workername=g_ci_w.getWorkersUuid('log')[0],
+workername=getWorkersDocker('log', worker_data_docker)[0],
workerbuilddir='builds',
collapseRequests=False,
canStartBuild=CanWorkerUpdateV,
@@ -71,7 +86,7 @@ def gentoo_builders(worker_data):
# Use multiplay workers
b.append(util.BuilderConfig(
name='build_request_data',
-workernames=g_ci_w.getWorkersUuid('local'),
+workernames=getWorkersLocal('local', worker_data_local),
collapseRequests=False,
factory=buildfactorys.build_request_check()
)
@@ -79,7 +94,7 @@ def gentoo_builders(worker_data):
# Use multiplay workers
b.append(util.BuilderConfig(
name='run_build_request',
-workernames=g_ci_w.getWorkersUuid('docker'),
+workernames=getWorkersDocker('build', worker_data_docker)
[gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
commit: 765bc21ded6f1c3003c877b60dc36235aab48f7f
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Sep 25 12:11:31 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Sep 25 12:11:31 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=765bc21d
Fix a typo in gitlab patch
Signed-off-by: Magnus Granberg gentoo.org>
patches/bb-gitlab.patch | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/patches/bb-gitlab.patch b/patches/bb-gitlab.patch
index f8e559d..8c466f8 100644
--- a/patches/bb-gitlab.patch
+++ b/patches/bb-gitlab.patch
@@ -101,7 +101,7 @@ index 8f2f80a83..40de0273e 100644
changes = [{
'author': f"{commit['author']['name']}
<{commit['author']['email']}>",
-'files': [], # @todo use rest API
-+'files': [], self._getFiles(attrs)
++'files': self._getFiles(attrs),
'comments': f"MR#{attrs['iid']}:
{attrs['title']}\n\n{attrs['description']}",
'revision': commit['id'],
'when_timestamp': when_timestamp,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 1a48ccb70732fa8367c36c842861c080c03eb1a2
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Sep 25 11:42:02 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Sep 25 11:42:02 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1a48ccb7
Add support for test on merge_request
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 10 ++
buildbot_gentoo_ci/steps/portage.py | 5 +
2 files changed, 15 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 3b6a8d0..00980e3 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1296,6 +1296,16 @@ class SetupStepts(BuildStep):
if build:
yield log.addStdout('Runing Build: YES\n')
aftersteps_list.append(RunBuild())
+# run tests
+test = False
+if self.getProperty('projectrepository_data')['test']:
+test = True
+if self.getProperty('projectrepository_data')['test_mr'] and
self.getProperty("event") == 'merge_request':
+test = True
+if test:
+yield log.addStdout('Runing Test: YES\n')
+else:
+yield log.addStdout('Runing Test: NO\n')
# run eclean pkg and dist
#if build:
#f.addStep(builders.RunEclean(step='pkg')
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index ff06c58..6d838d7 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -295,7 +295,12 @@ class SetPackageDefault(BuildStep):
yield log.addStdout(line + separator1)
# for test we need to add env and use
#FIXME: check restrictions, test use mask and required use
+test = False
if self.getProperty('projectrepository_data')['test']:
+test = True
+if self.getProperty('projectrepository_data')['test_mr'] and
self.getProperty("event") == 'merge_request':
+test = True
+if test:
auxdb_iuses = yield
self.gentooci.db.versions.getMetadataByUuidAndMatadata(self.getProperty("version_data")['uuid'],
'iuse')
for auxdb_iuse in auxdb_iuses:
iuse, status = getIUseValue(auxdb_iuse['value'])
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
commit: 3e54616897711b1958da178ca0578da08dd55cb3
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Sep 25 11:46:08 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Sep 25 11:46:08 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3e546168
Fix a typo on setStausBuilds
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/builds.py | 2 +-
buildbot_gentoo_ci/steps/builders.py | 2 +-
buildbot_gentoo_ci/steps/logs.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
index 6639ed5..57302ef 100644
--- a/buildbot_gentoo_ci/db/builds.py
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -54,7 +54,7 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
return self.db.pool.do(thd)
@defer.inlineCallbacks
-def setSatusBuilds(self, id, status):
+def setStausBuilds(self, id, status):
updated_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 00980e3..aed9edb 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -333,7 +333,7 @@ class SetupPropertys(BuildStep):
project_build_data = self.getProperty('project_build_data')
project_build_data['status'] = 'in-progress'
project_build_data['buildbot_build_id'] =
self.getProperty("buildnumber")
-yield self.gentooci.db.builds.setSatusBuilds(
+yield self.gentooci.db.builds.setStausBuilds(
project_build_data['id'],
project_build_data['status'])
yield self.gentooci.db.builds.setBuildbotBuildIdBuilds(
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 57f9394..becf7d1 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -596,7 +596,7 @@ class setBuildStatus(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
project_build_data = self.getProperty('project_build_data')
-yield self.gentooci.db.builds.setSatusBuilds(
+yield self.gentooci.db.builds.setStausBuilds(
project_build_data['id'],
self.getProperty('status')
)
[gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
commit: d81ea07c3ae4c6373ec2a798495481995ca15d7d
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Sep 25 11:25:32 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Sep 25 11:25:32 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d81ea07c
Update gitlab patch for 3.6.1
Signed-off-by: Magnus Granberg gentoo.org>
patches/bb-gitlab.patch | 184 +++-
1 file changed, 56 insertions(+), 128 deletions(-)
diff --git a/patches/bb-gitlab.patch b/patches/bb-gitlab.patch
index f57a2ac..f8e559d 100644
--- a/patches/bb-gitlab.patch
+++ b/patches/bb-gitlab.patch
@@ -1,5 +1,49 @@
a/buildbot/www/hooks/gitlab.py 2022-03-06 15:10:44.0 +0100
-+++ b/buildbot/www/hooks/gitlab.py 2022-08-07 22:22:16.570498909 +0200
+diff --git a/buildbot/reporters/gitlab.py b/buildbot/reporters/gitlab.py
+index 5a403d5d0..a6fc698e2 100644
+--- a/buildbot/reporters/gitlab.py
b/buildbot/reporters/gitlab.py
+@@ -169,8 +169,8 @@ class GitLabStatusPush(ReporterBase):
+ # FIXME: probably only want to report status for the last commit in
the changeset
+ for sourcestamp in sourcestamps:
+ sha = sourcestamp['revision']
+-if 'source_project_id' in props:
+-proj_id = props['source_project_id']
++if 'target_project_id' in props:
++proj_id = props['target_project_id']
+ else:
+ proj_id = yield self.getProjectId(sourcestamp)
+ if proj_id is None:
+@@ -191,8 +191,11 @@ class GitLabStatusPush(ReporterBase):
+ description=description
+ )
+ if res.code not in (200, 201, 204):
+-message = yield res.json()
+-message = message.get('message', 'unspecified error')
++if res.code.startswith("{"):
++message = yield res.json()
++message = message.get('message', 'unspecified error')
++else:
++message = res.code
+ log.msg(
+ f'Could not send status "{state}" for '
+ f'{sourcestamp["repository"]} at {sha}: {message}')
+diff --git a/buildbot/secrets/providers/file.py
b/buildbot/secrets/providers/file.py
+index 11daf327a..25e010b38 100644
+--- a/buildbot/secrets/providers/file.py
b/buildbot/secrets/providers/file.py
+@@ -55,7 +55,7 @@ class SecretInAFile(SecretProviderBase):
+ secretvalue = source.read()
+ if suffix:
+ secretfile = secretfile[:-len(suffix)]
+-if strip:
++if strip and not "OPENSSH PRIVATE KEY" in secretvalue:
+ secretvalue = secretvalue.rstrip("\r\n")
+ secrets[secretfile] = secretvalue
+ return secrets
+diff --git a/master/buildbot/www/hooks/gitlab.py
b/master/buildbot/www/hooks/gitlab.py
+index 8f2f80a83..40de0273e 100644
+--- a/buildbot/www/hooks/gitlab.py
b/buildbot/www/hooks/gitlab.py
@@ -19,6 +19,8 @@ import re
from dateutil.parser import parse as dateparse
@@ -9,19 +53,18 @@
from twisted.internet.defer import inlineCallbacks
from twisted.python import log
-@@ -26,6 +28,7 @@ from buildbot.process.properties import
- from buildbot.util import bytes2unicode
- from buildbot.www.hooks.base import BaseHookHandler
+@@ -28,6 +30,7 @@ from buildbot.www.hooks.base import BaseHookHandler
-+_HOSTED_BASE_URL = 'https://gitlab.com'
_HEADER_EVENT = b'X-Gitlab-Event'
_HEADER_GITLAB_TOKEN = b'X-Gitlab-Token'
++_HOSTED_BASE_URL = 'https://gitlab.com'
+
-@@ -94,6 +97,35 @@ class GitLabHandler(BaseHookHandler):
+ class GitLabHandler(BaseHookHandler):
+@@ -94,6 +97,34 @@ class GitLabHandler(BaseHookHandler):
return changes
-+#@inlineCallbacks
+def _configGitlabRest(self, token, baseURL=None):
+if baseURL is None:
+baseURL = _HOSTED_BASE_URL
@@ -53,82 +96,16 @@
def _process_merge_request_change(self, payload, event, codebase=None):
"""
Consumes the merge_request JSON as a python object and turn it into a
buildbot change.
-@@ -126,7 +158,7 @@ class GitLabHandler(BaseHookHandler):
+@@ -126,7 +157,7 @@ class GitLabHandler(BaseHookHandler):
changes = [{
'author': f"{commit['author']['name']}
<{commit['author']['email']}>",
-'files': [], # @todo use rest API
-+'files' : self._getFiles(attrs),
++'files': [], self._getFiles(attrs)
'comments': f"MR#{attrs['iid']}:
{attrs['title']}\n\n{attrs['description']}",
'revision': commit['id'],
'when_timestamp': when_timestamp,
-@@ -151,6 +183,65 @@ class GitLabHandler(BaseHookHandler):
- changes[0]['codebase'] = codebase
-
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/
commit: 01811de291a75c1b65b71575b33217b189509413 Author: Magnus Granberg gentoo org> AuthorDate: Sun Sep 11 21:48:32 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Sun Sep 11 21:48:32 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=01811de2 Fix a id error on id 950 Signed-off-by: Magnus Granberg gentoo.org> sql/search_pattern.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/search_pattern.sql b/sql/search_pattern.sql index 50fb4e1..afa87d4 100644 --- a/sql/search_pattern.sql +++ b/sql/search_pattern.sql @@ -747,7 +747,7 @@ INSERT INTO public.projects_pattern VALUES (919, 'e89c2c1a-46e0-4ded-81dd-c51afe -- ignore INSERT INTO public.projects_pattern VALUES (950, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', '^>>> /', 0, 0, 'ignore', 'ignore', 'search'); -INSERT INTO public.projects_pattern VALUES (950, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', '^./usr/', 0, 0, 'ignore', 'ignore', 'search'); +INSERT INTO public.projects_pattern VALUES (951, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', '^./usr/', 0, 0, 'ignore', 'ignore', 'search'); --
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: 0d19b4627eba83ffccadc753105b5cbdf53a68e7
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Sep 1 14:56:42 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Sep 1 14:56:42 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0d19b462
Add Bug ID for Irc reporter and nice titel
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/reporters.py | 7 ---
buildbot_gentoo_ci/steps/bugs.py | 24
buildbot_gentoo_ci/steps/logs.py | 16
3 files changed, 36 insertions(+), 11 deletions(-)
diff --git a/buildbot_gentoo_ci/config/reporters.py
b/buildbot_gentoo_ci/config/reporters.py
index 9832305..bd55037 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -8,10 +8,11 @@ from buildbot.reporters.message import MessageFormatter
from buildbot_gentoo_ci.reporters import irc
irc_template = '''{% set resultsList = ["\x0303SUCCESS", "\x0308WARNINGS",
"\x0304FAILURE"] %}\
-{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303"
}}repo/{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
+{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303"
}}Repo:{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
{{ build['properties']['revision'][0]|truncate(10, True) }} {{ "\x0302" }}{{
build['properties']['owners'][0][0] }}{{ "\x03" }} \
-{{ build['properties']['event'][0] }} {{ projects }}:{{
build['properties']['project_data'][0]['name'] }} \
-{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }}\
+{{ "\x0306" }}{{ build['properties']['event'][0] }}{{ "\x03" }} {{ projects
}}:{{ build['properties']['project_data'][0]['name'] }} \
+{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }} \
+{% if build['properties']['bgo'][0]['match'] is true %}{{ "\x0311" }}Bugid:
{{build['properties']['bgo'][0]['id']}}{{ "\x03" }}{% endif %}\
'''
def ircGenerators():
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 27b1388..801fc98 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -87,20 +87,28 @@ class GetBugs(BuildStep):
yield log.addStdout('Open Bugs\n')
match = False
for bug in buglist:
-yield log.addStdout('Bug: ' + str(bug['id']) + ' Summary: ' +
bug['summary'] +'\n')
-if re.search(self.getProperty('error_dict')['title_issue'][:20],
bug['summary']):
-print('Bug found')
-print(bug)
+yield log.addStdout(f"Bug: {str(bug['id'])} Summary:
{bug['summary']}\n")
+# we splite the lines to lists and try to match the words
+matches = 0
+match_search_text =
list(self.getProperty('error_dict')['title_issue'].split())
+match_bug_text = list(bug['summary'].split())
+#FIXME: add check for cp
+for match_word in match_search_text:
+if match_word in match_bug_text:
+matches = matches + 1
+if matches >= 10:
+print(f"Bug: {str(bug['id'])} Summary: {bug['summary']}")
match = {}
+match['match'] = True
match['id'] = bug['id']
match['summary'] = bug['summary']
+yield log.addStdout(f"Line to match:
{self.getProperty('error_dict')['title_issue']}\n")
if match:
-yield log.addStdout('Match bug found\n')
-yield log.addStdout('Bug: ' + str(match['id']) + ' Summary: ' +
match['summary'] +'\n')
+yield log.addStdout('Match bug: YES\n')
+yield log.addStdout(f"Bug: {str(match['id'])} Summary:
{match['summary']}\n")
self.setProperty("bgo", match, 'bgo')
return
-yield log.addStdout('NO Match bug found\n')
-self.setProperty("bgo", False, 'bgo')
+yield log.addStdout('Match bug: NO\n')
@defer.inlineCallbacks
def run(self):
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 164bb24..57f9394 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -91,6 +91,7 @@ class SetupPropertys(BuildStep):
else:
log_cpv = self.getProperty('cpv')
self.setProperty("log_cpv", log_cpv, 'log_cpv')
+self.setProperty("bgo", dict( match=False), 'bgo')
self.descriptionDone = 'Runing log checker on ' + log_cpv
return SUCCESS
@@ -333,6 +334,19 @@ class MakeIssue(BuildStep):
yield log.addStdout(log_cpv['full_logname'] + '\n')
yield log.addStdout('world.log' + '\n')
+def getNiceErrorLine(self, line):
+# strip away hex addresses, loong p
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
commit: 9e94e5e0e464ba589d25d60d6891c0a8dee0ecd7
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Aug 22 22:54:39 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Aug 22 22:54:39 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=9e94e5e0
Add Merge Requeste test to projects_repositorys in db
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/model.py| 1 +
buildbot_gentoo_ci/db/projects.py | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index be305f2..3f1dc0a 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -144,6 +144,7 @@ class Model(base.DBConnectorComponent):
sa.Column('pkgcheck', sa.Enum('package','full','none'),
default='none'),
sa.Column('build', sa.Boolean, default=False),
sa.Column('test', sa.Boolean, default=False),
+sa.Column('test_mr', sa.Boolean, default=False),
)
# projects etc/portage settings
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 8e2cd6f..9dcfa45 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -241,7 +241,8 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
auto=row.auto,
pkgcheck=pkgcheck,
build=row.build,
-test=row.test
+test=row.test,
+test_mr=row.test_mr
)
def _row2dict_projects_workers(self, conn, row):
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/
commit: e0aa266cf65e0fecd032ef58b188b71e51d5e340 Author: Magnus Granberg gentoo org> AuthorDate: Mon Aug 22 21:44:13 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Mon Aug 22 21:44:13 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e0aa266c Remove gitpuller from repositorys_type in db Signed-off-by: Magnus Granberg gentoo.org> sql/gentoo_ci_schema.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/gentoo_ci_schema.sql b/sql/gentoo_ci_schema.sql index f1725fa..70a7401 100644 --- a/sql/gentoo_ci_schema.sql +++ b/sql/gentoo_ci_schema.sql @@ -203,7 +203,6 @@ ALTER TYPE public.repositorys_mode OWNER TO postgres; -- CREATE TYPE public.repositorys_type AS ENUM ( -'gitpuller', 'git', 'gitlab' );
[gentoo-commits] proj/tinderbox-cluster:master commit in: /, patches/
commit: b635f1ab52fa63964ff9fc01f67638810c2fd2bf
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Aug 22 22:34:57 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Aug 22 22:34:57 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b635f1ab
Add buildbot gitlab patch
Signed-off-by: Magnus Granberg gentoo.org>
master.cfg | 21 +++--
patches/bb-gitlab.patch| 212 +
patches/secretstring.patch | 74
3 files changed, 225 insertions(+), 82 deletions(-)
diff --git a/master.cfg b/master.cfg
index af50ec2..dbdee3f 100644
--- a/master.cfg
+++ b/master.cfg
@@ -12,11 +12,13 @@ worker_data = [
{'uuid' : 'updatedb_2', 'password' : '', 'type' : 'local', 'enable' :
True, },
{'uuid' : 'updatedb_3', 'password' : '', 'type' : 'local', 'enable' :
True, },
{'uuid' : 'updatedb_4', 'password' : '', 'type' : 'local', 'enable' :
True, },
-{'uuid' : 'c89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'test',
'type' : 'log', 'enable' : True, },
-{'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'test',
'type' : 'docker', 'enable' : True, },
-{'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfb', 'password' : 'test',
'type' : 'docker', 'enable' : True, },
-{'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd', 'password' : 'test',
'type' : 'docker', 'enable' : True, },
-{'uuid' : 'c89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'test',
'type' : 'node', 'enable' : True, },
+{'uuid' : 'b89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'X', 'type'
: 'log', 'enable' : True, },
+{'uuid' : 'b89c2c1a-46e0-4ded-81dd-c51afeb7fcfb', 'password' : 'X', 'type'
: 'log', 'enable' : True, },
+{'uuid' : 'b89c2c1a-46e0-4ded-81dd-c51afeb7fcfc', 'password' : 'X', 'type'
: 'log', 'enable' : True, },
+{'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'X', 'type'
: 'docker', 'enable' : True, },
+{'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfb', 'password' : 'X', 'type'
: 'docker', 'enable' : True, },
+{'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd', 'password' : 'X', 'type'
: 'docker', 'enable' : True, },
+{'uuid' : 'c89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'X', 'type'
: 'node', 'enable' : True, },
]
# This is the dictionary that the buildmaster pays attention to. We also use
@@ -27,7 +29,8 @@ c['buildbotNetUsageData'] = None
### SECRETS
-c['secretsProviders'] =
[secrets.SecretInAFile(dirname="/var/lib/buildmaster/gentoo-ci-cloud/secrets/")]
+c['secretsProviders'] = []
+c['secretsProviders'].append(secrets.SecretInAFile(dirname="/var/lib/buildmaster/gentoo-ci-cloud/secrets"))
### WORKERS
@@ -88,7 +91,7 @@ c['titleURL'] = "https://gentoo-ci.gentoo.org";
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.
-c['buildbotURL'] = "http://90.231.13.235:8010/";
+c['buildbotURL'] = "http://0.0.0.0:8010/";
# minimalistic config to activate new web UI
c['www'] = dict(
@@ -97,6 +100,8 @@ c['www'] = dict(
change_hook_dialects={
'gitlab' : {
'secret': util.Secret("WWWHookGitlabToken"),
+'token': util.Secret("gitlabToken"),
+'baseUrl': 'https://gitlab.gentoo.org',
},
},
)
@@ -116,4 +121,4 @@ c['www']['ui_default_config'] = {
# It's easy to start with sqlite, but it's recommended to switch to a dedicated
# database, such as PostgreSQL or MySQL, for use in production environments.
#
http://docs.buildbot.net/current/manual/configuration/global.html#database-specification
-c['db_url'] = "postgresql://buildbot:fooo@192.168.1.9/buildbot"
+c['db_url'] = "postgresql://buildbot:X@192.0.0.0/buildbot"
diff --git a/patches/bb-gitlab.patch b/patches/bb-gitlab.patch
new file mode 100644
index 000..f57a2ac
--- /dev/null
+++ b/patches/bb-gitlab.patch
@@ -0,0 +1,212 @@
+--- a/buildbot/www/hooks/gitlab.py 2022-03-06 15:10:44.0 +0100
b/buildbot/www/hooks/gitlab.py 2022-08-07 22:22:16.570498909 +0200
+@@ -19,6 +19,8 @@ import re
+
+ from dateutil.parser import parse as dateparse
+
++import gitlab as python_gitlab
++
+ from twisted.internet.defer import inlineCallbacks
+ from twisted.python import log
+
+@@ -26,6 +28,7 @@ from buildbot.process.properties import
+ from buildbot.util import bytes2unicode
+ from buildbot.www.hooks.base import BaseHookHandler
+
++_HOSTED_BASE_URL = 'https://gitlab.com'
+ _HEADER_EVENT = b'X-Gitlab-Event'
+ _HEADER_GITLAB_TOKEN = b'X-Gitlab-Token'
+
+@@ -94,6 +97,35 @@ class GitLabHandler(BaseHookHandler):
+
+ return changes
+
++#@inlineCallbacks
++def _configGitlabRest(self, token, baseURL=None):
++if baseURL is None:
++baseURL = _HOSTED_BASE_URL
++if baseURL.endswith('/'):
++baseURL = baseURL[:-1]
++return python_gitlab.Gitlab(url=ba
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/
commit: 208581e198518f70fa2bfad0e01418eed8b345a1
Author: Magnus Granberg gentoo org>
AuthorDate: Sun Aug 21 08:43:00 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Sun Aug 21 08:43:00 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=208581e1
Add gentoo ci db schema
Signed-off-by: Magnus Granberg gentoo.org>
sql/gentoo_ci_schema.sql | 1525 ++
1 file changed, 1525 insertions(+)
diff --git a/sql/gentoo_ci_schema.sql b/sql/gentoo_ci_schema.sql
new file mode 100644
index 000..f1725fa
--- /dev/null
+++ b/sql/gentoo_ci_schema.sql
@@ -0,0 +1,1525 @@
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 13.3
+-- Dumped by pg_dump version 13.5
+
+-- Started on 2022-08-21 10:37:12 CEST
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
+SET check_function_bodies = false;
+SET xmloption = content;
+SET client_min_messages = warning;
+SET row_security = off;
+
+--
+-- TOC entry 2475 (class 1262 OID 155811)
+-- Name: gentoo-ci; Type: DATABASE; Schema: -; Owner: buildbot
+--
+
+CREATE DATABASE "gentoo-ci" WITH TEMPLATE = template0 ENCODING = 'UTF8' LOCALE
= 'sv_SE.utf8';
+
+
+ALTER DATABASE "gentoo-ci" OWNER TO buildbot;
+
+\connect -reuse-previous=on "dbname='gentoo-ci'"
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
+SET check_function_bodies = false;
+SET xmloption = content;
+SET client_min_messages = warning;
+SET row_security = off;
+
+--
+-- TOC entry 655 (class 1247 OID 155813)
+-- Name: projects_builds_status; Type: TYPE; Schema: public; Owner: buildbot
+--
+
+CREATE TYPE public.projects_builds_status AS ENUM (
+'failed',
+'completed',
+'in-progress',
+'waiting',
+'warning'
+);
+
+
+ALTER TYPE public.projects_builds_status OWNER TO buildbot;
+
+--
+-- TOC entry 658 (class 1247 OID 155824)
+-- Name: projects_pattern_search_type; Type: TYPE; Schema: public; Owner:
buildbot
+--
+
+CREATE TYPE public.projects_pattern_search_type AS ENUM (
+'in',
+'startswith',
+'endswith',
+'search'
+);
+
+
+ALTER TYPE public.projects_pattern_search_type OWNER TO buildbot;
+
+--
+-- TOC entry 661 (class 1247 OID 155834)
+-- Name: projects_pattern_status; Type: TYPE; Schema: public; Owner: buildbot
+--
+
+CREATE TYPE public.projects_pattern_status AS ENUM (
+'info',
+'warning',
+'error',
+'ignore'
+);
+
+
+ALTER TYPE public.projects_pattern_status OWNER TO buildbot;
+
+--
+-- TOC entry 664 (class 1247 OID 155844)
+-- Name: projects_pattern_type; Type: TYPE; Schema: public; Owner: buildbot
+--
+
+CREATE TYPE public.projects_pattern_type AS ENUM (
+'info',
+'qa',
+'compile',
+'configure',
+'install',
+'postinst',
+'prepare',
+'setup',
+'test',
+'unpack',
+'ignore',
+'issues',
+'misc',
+'elog',
+'pretend'
+);
+
+
+ALTER TYPE public.projects_pattern_type OWNER TO buildbot;
+
+--
+-- TOC entry 667 (class 1247 OID 155868)
+-- Name: projects_portage_directorys; Type: TYPE; Schema: public; Owner:
buildbot
+--
+
+CREATE TYPE public.projects_portage_directorys AS ENUM (
+'make.profile',
+'repos.conf'
+);
+
+
+ALTER TYPE public.projects_portage_directorys OWNER TO buildbot;
+
+--
+-- TOC entry 761 (class 1247 OID 550221)
+-- Name: projects_portage_package_directorys; Type: TYPE; Schema: public;
Owner: buildbot
+--
+
+CREATE TYPE public.projects_portage_package_directorys AS ENUM (
+'use',
+'accept_keywords',
+'exclude',
+'env'
+);
+
+
+ALTER TYPE public.projects_portage_package_directorys OWNER TO buildbot;
+
+--
+-- TOC entry 670 (class 1247 OID 155874)
+-- Name: projects_repositorys_pkgcheck; Type: TYPE; Schema: public; Owner:
buildbot
+--
+
+CREATE TYPE public.projects_repositorys_pkgcheck AS ENUM (
+'package',
+'full',
+'none'
+);
+
+
+ALTER TYPE public.projects_repositorys_pkgcheck OWNER TO buildbot;
+
+--
+-- TOC entry 673 (class 1247 OID 155882)
+-- Name: projects_status; Type: TYPE; Schema: public; Owner: buildbot
+--
+
+CREATE TYPE public.projects_status AS ENUM (
+'stable',
+'all',
+'unstable'
+);
+
+
+ALTER TYPE public.projects_status OWNER TO buildbot;
+
+--
+-- TOC entry 777 (class 1247 OID 1708870)
+-- Name: repositorys_method; Type: TYPE; Schema: public; Owner: postgres
+--
+
+CREATE TYPE public.repositorys_method AS ENUM (
+'clobber',
+'fresh',
+'clean',
+'copy'
+);
+
+
+ALTER TYPE public.repositorys_method OWNER TO postgres;
+
+--
+-- TOC entry 774 (class 1247 OID 1708856)
+-- Name: repositorys_mode; Type: TYPE; Schema: public; Owner: postgres
+--
+
+CREATE TYPE pub
[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/
commit: 68ecdf96df4a062728672ecad2e275f7c17ee388
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Aug 18 14:39:06 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Aug 18 14:46:53 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=68ecdf96
Fix line 14 on bb worker default docker file Thanx FurudeRika
Signed-off-by: Magnus Granberg gentoo.org>
docker/GentooBuildbotWorkerDefault.Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docker/GentooBuildbotWorkerDefault.Dockerfile
b/docker/GentooBuildbotWorkerDefault.Dockerfile
index d1197c6..a7c7af1 100644
--- a/docker/GentooBuildbotWorkerDefault.Dockerfile
+++ b/docker/GentooBuildbotWorkerDefault.Dockerfile
@@ -11,7 +11,7 @@ COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo
# emerge needed deps buildbot-worker, psycopg, git and sqlalchemy
# get the needed buildbot-worker config
RUN echo -e "[binhost]\npriority = \nsync-uri =
https://gentoo.osuosl.org/experimental/amd64/binpkg/default/linux/17.1/x86-64/\n";
| cat >> /etc/portage/binrepos.conf\
- && echo 'EMERGE_DEFAULT_OPTS="--binpkg-respect-use=n --usepkg=y --getbinpkg=y
--autounmask-write --autounmask-continue --autounmask-keep-keywords=y
--autounmask-use=y"' | cat >> /etc/portage/m>
+ && echo 'EMERGE_DEFAULT_OPTS="--binpkg-respect-use=n --usepkg=y --getbinpkg=y
--autounmask-write --autounmask-continue --autounmask-keep-keywords=y
--autounmask-use=y"' | cat >> /etc/portage/make.conf\
&& echo 'FEATURES="-ipc-sandbox -pid-sandbox -network-sandbox -usersandbox
-mount-sandbox sandbox"' | cat >> /etc/portage/make.conf\
&& echo 'FEATURES="${FEATURES} parallel-install parallel-fetch -merge-sync"'
| cat >> /etc/portage/make.conf\
&& echo 'FEATURES="${FEATURES} buildpkg"' | cat >> /etc/portage/make.conf\
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 25877447fec0f718c22bdc7cd6f31423aae75f7e
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Aug 10 21:51:13 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Aug 10 21:51:13 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=25877447
Set head to 10 for EAPI search
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index dbeaf14..26cf9fc 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -436,7 +436,7 @@ class SetupStepsForCheckV(BuildStep):
name =
'RunGetEAPI',
haltOnFailure =
True,
flunkOnFailure =
True,
-command=['head',
'-n', '8', self.getProperty("ebuild_file")],
+command=['head',
'-n', '10', self.getProperty("ebuild_file")],
strip=False,
extract_fn=PersOutputOfGetEapi
))
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/
commit: 26929ff836acfe559456bed9f400dd4879973d8b
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Aug 10 21:46:50 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Aug 10 21:46:50 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=26929ff8
Use one scheduler for push and mr
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/schedulers.py | 26 --
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index b769f00..60f1857 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -42,9 +42,17 @@ def getGitChanges(props):
change_data['project'] = k['project']
return change_data
-def mr_branch_fn(branch):
-# if branch end with -mr or -pr
-if branch[-3] == '-mr' or branch[-3] == '-pr':
+def bb_branch_fn(branch):
+# check branch
+print(f"Branch: {branch}")
+if branch.endswith('-mr') or branch.endswith('-pr') or branch == 'master':
+return True
+return False
+
+def bb_category_fn(category):
+# check event
+print(f"Category: {category}")
+if category == 'push' or category == 'merge_request':
return True
return False
@@ -56,16 +64,7 @@ def gentoo_schedulers():
change_data = getGitChanges
),
builderNames = builderUpdateDbNames,
-change_filter=util.ChangeFilter(branch='master', category='push'),
-)
-scheduler_update_db_mr= schedulers.SingleBranchScheduler(
-name='scheduler_update_db_mr',
-treeStableTimer=0,
-properties = dict(
-change_data = getGitChanges
-),
-builderNames = builderUpdateDbNames,
-change_filter=util.ChangeFilter(branch_fn=mr_branch_fn,
category='merge_request'),
+change_filter=util.ChangeFilter(branch_fn=bb_branch_fn,
category_fn=bb_category_fn),
)
create_stage4 = schedulers.ForceScheduler(
name="create_stage4",
@@ -102,7 +101,6 @@ def gentoo_schedulers():
s = []
s.append(create_stage4)
s.append(scheduler_update_db)
-s.append(scheduler_update_db_mr)
s.append(update_repo_check)
s.append(update_cpv_data)
s.append(update_v_data)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 3791e957fd2bb28f350f0ff7399c0c6e43b561d6
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Aug 5 18:32:11 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Aug 5 18:32:11 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3791e957
Add haskell-updater for haskell packages
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 147 ++-
1 file changed, 91 insertions(+), 56 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index a54c68c..3b6a8d0 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,6 +4,7 @@
import os
import re
import json
+from pathlib import Path
from portage.versions import catpkgsplit, cpv_getversion
from portage.dep import dep_getcpv, dep_getslot, dep_getrepo
@@ -90,6 +91,8 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# split the lines
#FIXME: Handling of stderr output
stderr_line_list = []
+if stderr != '' and not emerge_output['failed']:
+emerge_output['failed'] = True
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
@@ -340,6 +343,7 @@ class SetupPropertys(BuildStep):
print(self.getProperty("project_build_data"))
self.masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber")))
self.setProperty('masterdest', self.masterdest, 'masterdest')
+self.setProperty('build_env', {}, 'build_env')
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for
project', self.getProperty('project_data')['name']])
return SUCCESS
@@ -355,7 +359,6 @@ class RunEmerge(BuildStep):
super().__init__(**kwargs)
self.descriptionSuffix = self.step
self.name = 'Setup emerge for ' + self.step + ' step'
-self.build_env = {}
self.build_timeout = 0
@defer.inlineCallbacks
@@ -374,25 +377,28 @@ class RunEmerge(BuildStep):
aftersteps_list = []
#FIXME: Set build timeout in config
self.build_timeout = 6600
-# set env
-# https://bugs.gentoo.org/683118
-# export TERM=linux
-# export TERMINFO=/etc/terminfo
-self.build_env['TERM'] = 'linux'
-self.build_env['TERMINFO'] = '/etc/terminfo'
-# Lang
-self.build_env['LANG'] = 'C.utf8'
-self.build_env['LC_MESSAGES'] = 'C'
-# no color
-self.build_env['CARGO_TERM_COLOR'] = 'never'
-self.build_env['GCC_COLORS'] = '0'
-self.build_env['OCAML_COLOR'] = 'never'
-self.build_env['PY_FORCE_COLOR'] = '0'
-self.build_env['PYTEST_ADDOPTS'] = '--color=no'
-self.build_env['NO_COLOR'] = '1'
-# not all terms support urls
-self.build_env['GCC_URLS'] = 'no'
-self.build_env['TERM_URLS'] = 'no'
+if self.step == 'update':
+# set env
+build_env = {}
+# https://bugs.gentoo.org/683118
+# export TERM=linux
+# export TERMINFO=/etc/terminfo
+build_env['TERM'] = 'linux'
+build_env['TERMINFO'] = '/etc/terminfo'
+# Lang
+build_env['LANG'] = 'C.utf8'
+build_env['LC_MESSAGES'] = 'C'
+# no color
+build_env['CARGO_TERM_COLOR'] = 'never'
+build_env['GCC_COLORS'] = '0'
+build_env['OCAML_COLOR'] = 'never'
+build_env['PY_FORCE_COLOR'] = '0'
+build_env['PYTEST_ADDOPTS'] = '--color=no'
+build_env['NO_COLOR'] = '1'
+# not all terms support urls
+build_env['GCC_URLS'] = 'no'
+build_env['TERM_URLS'] = 'no'
+self.setProperty('build_env', build_env, 'build_env')
if self.step == 'pre-update':
shell_commad_list.append('-uDN')
@@ -432,6 +438,7 @@ class RunEmerge(BuildStep):
strip=True,
extract_fn=PersOutputOfEmerge,
workdir='/',
+env=self.getProperty("build_env"),
timeout=self.build_timeout
))
aftersteps_list.append(CheckEmergeLogs('update'))
@@ -548,7 +555,7 @@ class RunEmerge(BuildStep):
strip=True,
extract_fn=PersOutputOfEmerge,
workdir='/',
-env=self.build_env,
+env=self.getProperty("build_env"),
timeout=self.build_timeout
))
aftersteps_list.append(CheckEmergeLogs('build'))
@@ -682,11 +689,9 @@ class CheckEmergeLogs(BuildStep):
@defer.inlineCallbacks
def createDistDir(self):
workdir = yield os.path.join(self.master.b
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/
commit: 457eadb511be7e44744cfa4a91a302614b5d754f Author: Magnus Granberg gentoo org> AuthorDate: Sat Jul 30 22:43:32 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Sat Jul 30 22:43:32 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=457eadb5 Add scheduler_update_db_mr for mr/pr support Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/schedulers.py | 19 ++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py index 2e5379c..b769f00 100644 --- a/buildbot_gentoo_ci/config/schedulers.py +++ b/buildbot_gentoo_ci/config/schedulers.py @@ -39,8 +39,15 @@ def getGitChanges(props): change_data['revision'] = k['revision'] change_data['timestamp'] = k['when_timestamp'] change_data['branch'] = k['branch'] +change_data['project'] = k['project'] return change_data +def mr_branch_fn(branch): +# if branch end with -mr or -pr +if branch[-3] == '-mr' or branch[-3] == '-pr': +return True +return False + def gentoo_schedulers(): scheduler_update_db = schedulers.SingleBranchScheduler( name='scheduler_update_db', @@ -49,7 +56,16 @@ def gentoo_schedulers(): change_data = getGitChanges ), builderNames = builderUpdateDbNames, -change_filter=util.ChangeFilter(branch='master'), +change_filter=util.ChangeFilter(branch='master', category='push'), +) +scheduler_update_db_mr= schedulers.SingleBranchScheduler( +name='scheduler_update_db_mr', +treeStableTimer=0, +properties = dict( +change_data = getGitChanges +), +builderNames = builderUpdateDbNames, +change_filter=util.ChangeFilter(branch_fn=mr_branch_fn, category='merge_request'), ) create_stage4 = schedulers.ForceScheduler( name="create_stage4", @@ -86,6 +102,7 @@ def gentoo_schedulers(): s = [] s.append(create_stage4) s.append(scheduler_update_db) +s.append(scheduler_update_db_mr) s.append(update_repo_check) s.append(update_cpv_data) s.append(update_v_data)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 3f7cd0136414f731cb97d50c486c291776d16830 Author: Magnus Granberg gentoo org> AuthorDate: Sat Jul 30 22:40:02 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Sat Jul 30 22:40:02 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3f7cd013 Fix UpdateRepos step in nodes Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/nodes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py index 3e15e5f..902f7a8 100644 --- a/buildbot_gentoo_ci/steps/nodes.py +++ b/buildbot_gentoo_ci/steps/nodes.py @@ -22,7 +22,7 @@ from buildbot.process.results import SKIPPED from buildbot.plugins import steps, util from buildbot_gentoo_ci.steps import portage as portage_steps -from buildbot_gentoo_ci.steps import builders as builders_steps +from buildbot_gentoo_ci.steps import repos as repos_steps class SetupPropertys(BuildStep): name = 'Setup propertys for stage4 image' @@ -87,7 +87,7 @@ class SetupStage4Steps(BuildStep): # download stage3 aftersteps_list.append(GetSteg3()) # setup portage -aftersteps_list.append(builders_steps.UpdateRepos()) +aftersteps_list.append(repos_steps.UpdateRepos()) aftersteps_list.append(portage_steps.SetReposConf()) aftersteps_list.append(portage_steps.SetMakeConf()) # add localegen
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/
commit: 795fec9c1b421559ade5b82fc7232384d31a02d2
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Jul 28 11:35:00 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Jul 28 11:35:00 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=795fec9c
Add support branch,event and gitlab/github project name on IRC reporter
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/reporters.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/config/reporters.py
b/buildbot_gentoo_ci/config/reporters.py
index d37dd2a..9832305 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -8,9 +8,9 @@ from buildbot.reporters.message import MessageFormatter
from buildbot_gentoo_ci.reporters import irc
irc_template = '''{% set resultsList = ["\x0303SUCCESS", "\x0308WARNINGS",
"\x0304FAILURE"] %}\
-{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303"
}}repo/{{ projects }}{{ "\x03" }} \
+{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303"
}}repo/{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
{{ build['properties']['revision'][0]|truncate(10, True) }} {{ "\x0302" }}{{
build['properties']['owners'][0][0] }}{{ "\x03" }} \
-{{ build['properties']['project_data'][0]['name'] }} \
+{{ build['properties']['event'][0] }} {{ projects }}:{{
build['properties']['project_data'][0]['name'] }} \
{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }}\
'''
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
commit: 60ce5fec404770c2dbc58352808397f7e9b87c93
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Jul 28 11:31:07 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Jul 28 11:31:07 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=60ce5fec
Add support for gitlab/github project name in project db
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/db/model.py| 2 ++
buildbot_gentoo_ci/db/projects.py | 1 +
2 files changed, 3 insertions(+)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index d6e9860..be305f2 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -109,6 +109,8 @@ class Model(base.DBConnectorComponent):
default=lambda: str(uuid.uuid4())),
# project's name
sa.Column('name', sa.String(255), nullable=False),
+# project name on gitlab/github
+sa.Column('git_project_name', sa.String(255), nullable=False),
# description of the project
sa.Column('description', sa.Text, nullable=True),
sa.Column('profile', sa.String(255), nullable=False),
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 936cd9d..8e2cd6f 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -216,6 +216,7 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
return dict(
uuid=row.uuid,
name=row.name,
+git_project_name=row.git_project_name,
description=row.description,
profile=row.profile,
profile_repository_uuid=row.profile_repository_uuid,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
commit: aba0962b8f901301302a2fefacac14f8cd34a51b
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 27 19:32:09 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 27 19:32:09 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aba0962b
Move UpdateRepos to repos.py and support gitlab stage
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 4 +-
buildbot_gentoo_ci/db/model.py | 12 -
buildbot_gentoo_ci/db/repositorys.py | 19 +++-
buildbot_gentoo_ci/steps/builders.py | 40
buildbot_gentoo_ci/steps/repos.py | 73 +-
5 files changed, 102 insertions(+), 46 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 67447aa..7f2dff9 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -46,7 +46,7 @@ def update_db_cpv():
# set needed Propertys
f.addStep(package.SetupPropertys())
# update the repositorys listed in project_repository
-f.addStep(builders.UpdateRepos())
+f.addStep(repos.UpdateRepos())
# add repo.conf
#f.addStep(portage.SetReposConf())
# if categorys in db
@@ -99,7 +99,7 @@ def run_build_request():
# set needed Propertys
f.addStep(builders.SetupPropertys())
# update the repositorys listed in project_repository
-f.addStep(builders.UpdateRepos())
+f.addStep(repos.UpdateRepos())
# Clean and add new /etc/portage
#NOTE: remove the symlink befor the dir
#f.addStep(buildbot_steps.ShellCommand(
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 908ebb5..d6e9860 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -71,10 +71,18 @@ class Model(base.DBConnectorComponent):
# description of the repository
sa.Column('description', sa.Text, nullable=True),
sa.Column('url', sa.String(255), nullable=True),
-sa.Column('type', sa.Enum('gitpuller'), nullable=False,
default='gitpuller'),
+sa.Column('branch', sa.String(255), nullable=True),
+sa.Column('type', sa.Enum('git', 'gitlab'), nullable=False,
default='git'),
+sa.Column('mode', sa.Enum('full', 'incremental'), nullable=False,
default='incremental'),
+sa.Column('method', sa.Enum('clobber', 'fresh', 'clean', 'copy'),
nullable=False, default='fresh'),
+sa.Column('alwaysuselatest', sa.Boolean, default=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
-sa.Column('ebuild', sa.Boolean, default=False),
+sa.Column('ebuild', sa.Boolean, default=True),
+sa.Column('merge', sa.Boolean, default=False),
+# filenames for SecretInAFile
+sa.Column('sshprivatekey', sa.String(50), nullable=True),
+sa.Column('sshhostkey', sa.String(50), nullable=True),
)
# Use by GitPoller
diff --git a/buildbot_gentoo_ci/db/repositorys.py
b/buildbot_gentoo_ci/db/repositorys.py
index 8bac1a1..4d5d048 100644
--- a/buildbot_gentoo_ci/db/repositorys.py
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -91,15 +91,32 @@ class
RepositorysConnectorComponent(base.DBConnectorComponent):
yield self.db.pool.do(thd)
def _row2dict(self, conn, row):
+if row.branch == 'none':
+branch = False
+else:
+branch = row.branch
+if row.sshprivatekey == 'none':
+sshprivatekey = False
+sshhostkey = False
+else:
+sshprivatekey = row.sshprivatekey
+sshhostkey = row.sshhostkey
return dict(
uuid=row.uuid,
name=row.name,
description=row.description,
url=row.url,
+type=row.type,
+branch=branch,
+mode=row.mode,
+method=row.method,
+alwaysuselatest=row.alwaysuselatest,
auto=row.auto,
enabled=row.enabled,
ebuild=row.ebuild,
-type=row.type
+merge=row.merge,
+sshprivatekey=sshprivatekey,
+sshhostkey=sshhostkey
)
def _row2dict_gitpuller(self, conn, row):
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 383ae60..a54c68c 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -343,46 +343,6 @@ class SetupPropertys(BuildStep):
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for
project', self.getProperty('project_data')['name']])
return SUCCESS
-# Should be moved to repos.py
-class UpdateRepos(BuildStep):
-
-name = 'UpdateRepos'
-description = 'Running'
-descriptionDone = 'Ran'
-descriptionSuffix = None
-haltOn
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 843b4272bf91ac8cf135664cab48c515f23d6cad
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jul 26 12:47:11 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jul 26 12:47:11 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=843b4272
Use rootworkdir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/nodes.py
b/buildbot_gentoo_ci/steps/nodes.py
index a77c878..3e15e5f 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -76,6 +76,7 @@ class SetupStage4Steps(BuildStep):
workerdest = yield
os.path.join(self.getProperty("workerbase"), self.getProperty('project_uuid'))
workerdest_etc = yield os.path.join(workerdest, 'etc')
print(workerdest_etc)
+self.setProperty('rootworkdir', workerdest, 'rootworkdir')
# create dir
aftersteps_list.append(steps.ShellCommand(
flunkOnFailure=True,
@@ -86,9 +87,9 @@ class SetupStage4Steps(BuildStep):
# download stage3
aftersteps_list.append(GetSteg3())
# setup portage
-
aftersteps_list.append(builders_steps.UpdateRepos(workdir=workerdest))
-
aftersteps_list.append(portage_steps.SetReposConf(workdir=workerdest))
-
aftersteps_list.append(portage_steps.SetMakeConf(workdir=workerdest))
+aftersteps_list.append(builders_steps.UpdateRepos())
+aftersteps_list.append(portage_steps.SetReposConf())
+aftersteps_list.append(portage_steps.SetMakeConf())
# add localegen
#FIXME: set that in config
locale_conf = []
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 63ea1fb9235a4585754d5b8acc41c281596ab4fc
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jul 26 12:57:35 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jul 26 12:57:35 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=63ea1fb9
Clean up and add use of rootworkdir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 204
1 file changed, 21 insertions(+), 183 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 431fe9a..ff06c58 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -20,6 +20,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
+from buildbot.process import remotecommand
from buildbot.plugins import steps
from buildbot_gentoo_ci.steps import master as master_steps
@@ -34,39 +35,6 @@ def WriteTextToFile(path, text_list):
yield f.write(separator)
yield f.close
-def PersOutputOfEbuildSH(rc, stdout, stderr):
-metadata = None
-NoSplit = []
-NoSplit.append('DESCRIPTION')
-#make dict of the stout
-index = 1
-metadata_line_dict = {}
-for text_line in stdout.splitlines():
-metadata_line_dict[index] = text_line
-index = index + 1
-# should have 22 lines
-if len(auxdbkeys) != index -1:
-# number of lines is incorrect.
-return {
-'auxdb' : metadata
-}
-# split all keys to list instead of speces
-metadata = {}
-i = 1
-for key in auxdbkeys:
-if metadata_line_dict[i] == '':
-metadata[key] = None
-else:
-if ' ' in metadata_line_dict[i] and key not in NoSplit:
-metadata[key] = metadata_line_dict[i].split(' ')
-else:
-metadata[key] = []
-metadata[key].append(metadata_line_dict[i])
-i = i + 1
-return {
-'auxdb' : metadata
-}
-
class SetMakeProfile(BuildStep):
name = 'SetMakeProfile'
@@ -76,8 +44,7 @@ class SetMakeProfile(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, workdir=False, **kwargs):
-self.rootworkdir = workdir
+def __init__(self, **kwargs):
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -105,8 +72,8 @@ class SetMakeProfile(BuildStep):
'ln',
'-s'
]
-if self.rootworkdir:
-symlink_makeprofile_path = yield os.path.join(self.rootworkdir,
'etc/portage/make.profile')
+if self.getProperty('rootworkdir'):
+symlink_makeprofile_path = yield
os.path.join(self.getProperty('rootworkdir'), 'etc/portage/make.profile')
else:
symlink_makeprofile_path = '/etc/portage/make.profile'
shell_commad_list.append(makeprofile_path)
@@ -130,8 +97,7 @@ class SetReposConf(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, workdir=False, **kwargs):
-self.rootworkdir = workdir
+def __init__(self, **kwargs):
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -139,8 +105,8 @@ class SetReposConf(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
portage_repos_path = self.getProperty('portage_repos_path')
project_data = self.getProperty('project_data')
-if self.rootworkdir:
-portage_etc_path = yield os.path.join(self.rootworkdir,
'etc/portage/')
+if self.getProperty('rootworkdir'):
+portage_etc_path = yield
os.path.join(self.getProperty('rootworkdir'), 'etc/portage/')
else:
portage_etc_path = '/etc/portage/'
# setup the default.conf
@@ -148,6 +114,13 @@ class SetReposConf(BuildStep):
if repos_conf_data is None:
print('Default repo is not set in repos.conf')
return FAILURE
+self.aftersteps_list = []
+# check if we have the dir
+cmd = remotecommand.RemoteCommand('stat', {'file': portage_etc_path +
'repos.conf'})
+yield self.runCommand(cmd)
+if cmd.didFail():
+self.aftersteps_list.append(steps.MakeDirectory(dir="repos.conf",
+workdir=portage_etc_path))
log = yield self.addLog('repos.conf')
# check if repos_conf_data['value'] is vaild repo name
separator = '\n'
@@ -156,11 +129,11 @@ class SetReposConf(BuildStep):
default_conf.append('main-repo = ' + repos_conf_data['value'])
default_conf.append('auto-sync = no')
default_conf_string = separator.join(default_conf)
-yield self.build.addStepsAfterCurrentStep([
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 8ab6bf127155a3bddca43767fb7aa208a3ea4fe7
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jul 26 12:59:19 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jul 26 12:59:19 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8ab6bf12
Add support for use of rootworkdir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 9 +
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 38dfa79..383ae60 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -314,6 +314,7 @@ class SetupPropertys(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
self.setProperty('portage_repos_path',
self.gentooci.config.project['project']['worker_portage_repos_path'],
'portage_repos_path')
+self.setProperty('rootworkdir', False, 'rootworkdir')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
project_data = yield
self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
@@ -342,6 +343,7 @@ class SetupPropertys(BuildStep):
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for
project', self.getProperty('project_data')['name']])
return SUCCESS
+# Should be moved to repos.py
class UpdateRepos(BuildStep):
name = 'UpdateRepos'
@@ -351,8 +353,7 @@ class UpdateRepos(BuildStep):
haltOnFailure = True
flunkOnFailure = True
-def __init__(self, workdir=False, **kwargs):
-self.rootworkdir = workdir
+def __init__(self, **kwargs):
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -364,8 +365,8 @@ class UpdateRepos(BuildStep):
projects_repositorys_data = yield
self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
for project_repository_data in projects_repositorys_data:
repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
-if self.rootworkdir:
-repository_path = os.path.join(self.rootworkdir,
portage_repos_path[1:], repository_data['name'])
+if self.getProperty('rootworkdir'):
+repository_path =
os.path.join(self.getProperty('rootworkdir'), portage_repos_path[1:],
repository_data['name'])
else:
repository_path = os.path.join(portage_repos_path,
repository_data['name'], '')
yield self.build.addStepsAfterCurrentStep([
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 59e77c4f9f2ca0d05aa4945e3e765353734791b1
Author: Magnus Granberg gentoo org>
AuthorDate: Tue Jul 26 12:53:07 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Tue Jul 26 12:53:07 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=59e77c4f
Add SetupPropertys for use of rootworkdir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/package.py | 18 ++
1 file changed, 18 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/package.py
b/buildbot_gentoo_ci/steps/package.py
index 76c90cc..81b400d 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -16,6 +16,23 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.plugins import steps
+class SetupPropertys(BuildStep):
+name = 'Setup propertys for CPV check'
+description = 'Running'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+# set this in config
+super().__init__(**kwargs)
+
+#@defer.inlineCallbacks
+def run(self):
+self.setProperty('portage_repos_path', '/repositorys',
'portage_repos_path')
+self.setProperty('rootworkdir', '/var/lib/buildbot_worker',
'rootworkdir')
+return SUCCESS
+
class AddPackage(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
@@ -100,6 +117,7 @@ class TriggerCheckForV(BuildStep):
'category_data' : self.getProperty("category_data"),
'change_data' : self.getProperty("change_data"),
'project_data' : self.getProperty("project_data"),
+'cp_worker' : self.getProperty('workername'),
}
)
)
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, docker/
commit: 1592e1d7979e55f71c676869c1d3cc1a21064860
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 27 11:01:45 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 27 11:01:45 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1592e1d7
Use worker for Version check (get aux db)
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/builders.py | 24 ++-
buildbot_gentoo_ci/config/buildfactorys.py| 52 +++---
buildbot_gentoo_ci/config/workers.py | 18 ++-
buildbot_gentoo_ci/steps/version.py | 225 --
docker/GentooBuildbotWorkerDefault.Dockerfile | 16 +-
5 files changed, 241 insertions(+), 94 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index e168cf5..82552fa 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -20,6 +20,14 @@ def CanWorkerBuildProject(builder, wfb, request):
print('no worker')
return False
+# Use same worker as update_cpv_data was done by so we have same git commit
+def CanWorkerUpdateV(builder, wfb, request):
+print(request.properties['cp_worker'])
+print(wfb)
+if wfb.worker.workername == request.properties['cp_worker']:
+return True
+return False
+
def gentoo_builders(worker_data):
b = []
g_ci_w = gentoo_ci_workers(worker_data)
@@ -39,24 +47,24 @@ def gentoo_builders(worker_data):
factory=buildfactorys.update_repo_check()
)
)
-# Use multiplay workers depend on Property(cp)
-# if cp do not match next one, use diffrent worker then
-# or first cp have done its buildsteps.
-# first LocalWorker need to be done before we can use mulitplay workers
(git pull)
+# update cpv in db and call build_request_data
+#FIXME: look so we don't run parallel with diffrent worker
+# (builders.UpdateRepos step)
b.append(util.BuilderConfig(
name='update_cpv_data',
-workernames=g_ci_w.getWorkersUuid('local'),
+workernames=g_ci_w.getWorkersUuid('log')[0],
workerbuilddir='builds',
collapseRequests=False,
-factory=buildfactorys.update_db_cp()
+factory=buildfactorys.update_db_cpv()
)
)
# Use multiplay workers
b.append(util.BuilderConfig(
name='update_v_data',
-workernames=g_ci_w.getWorkersUuid('local'),
+workername=g_ci_w.getWorkersUuid('log')[0],
workerbuilddir='builds',
collapseRequests=False,
+canStartBuild=CanWorkerUpdateV,
factory=buildfactorys.update_db_v()
)
)
@@ -80,7 +88,7 @@ def gentoo_builders(worker_data):
# Use multiplay workers
b.append(util.BuilderConfig(
name='parse_build_log',
-workernames=g_ci_w.getWorkersUuid('log'),
+workernames=g_ci_w.getWorkersUuid('log')[1:],
collapseRequests=False,
factory=buildfactorys.parse_build_log()
)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index d07e31b..67447aa 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -22,12 +22,13 @@ def update_db_check():
# return profile_repository, project
f.addStep(update_db.GetDataGentooCiProject())
# update the repos
-f.addStep(update_db.TriggerUpdateRepositorys())
+#f.addStep(update_db.TriggerUpdateRepositorys())
# Make a for loop and trigger new builders with cpv from git_changes
# return cpv, repository, project_data
f.addStep(update_db.TriggerCheckForCPV())
return f
+# we run repo update in update_db_cpv instead
def update_repo_check():
f = util.BuildFactory()
# FIXME: 6
@@ -40,9 +41,14 @@ def update_repo_check():
f.addStep(repos.CheckRepository())
return f
-def update_db_cp():
+def update_db_cpv():
f = util.BuildFactory()
-# FIXME: 2
+# set needed Propertys
+f.addStep(package.SetupPropertys())
+# update the repositorys listed in project_repository
+f.addStep(builders.UpdateRepos())
+# add repo.conf
+#f.addStep(portage.SetReposConf())
# if categorys in db
# return category_data
# add check category path step at end
@@ -57,33 +63,25 @@ def update_db_cp():
# add package to db step
# return package_data
f.addStep(package.CheckP())
-# Trigger new builders with v from cpv
-# return package_data, cpv, repository_data, project_data
+# Trigger update_db_v
f.addStep(package.TriggerCheckForV())
return f
def update_db_v():
f = util.BuildFactory()
-# FIXME: 3
-# if version in db
-# return version_data
-f.addStep(version.GetVData())
-# check path and hash
-f.addStep(version.CheckPathHash())
-# if not path
-# if not hash
-# add delet
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: ae8716fd198fb5502781e4e6147f59bb19361b21 Author: Magnus Granberg gentoo org> AuthorDate: Tue Jul 26 11:53:52 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Tue Jul 26 11:53:52 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ae8716fd Set 1H for timeout Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/logs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py index e139cf9..164bb24 100644 --- a/buildbot_gentoo_ci/steps/logs.py +++ b/buildbot_gentoo_ci/steps/logs.py @@ -120,7 +120,7 @@ class SetupParserBuildLoger(BuildStep): self.aftersteps_list.append(steps.FileDownload( mastersrc=mastersrc_log, workerdest=log_cpv['full_logname'] -)) +)) # Upload log parser py code self.aftersteps_list.append(steps.FileDownload( mastersrc=mastersrc_py, @@ -145,7 +145,8 @@ class SetupParserBuildLoger(BuildStep): flunkOnFailure = True, command=command, strip=False, - extract_fn=PersOutputOfLogParser + extract_fn=PersOutputOfLogParser, +timeout=3600 )) yield self.build.addStepsAfterCurrentStep(self.aftersteps_list) return SUCCESS
[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
commit: 07a0c6eb02b632e88bc8d288c33015c8b43e93e0
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Jul 22 22:18:44 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Jul 22 22:18:44 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=07a0c6eb
Use Webhook for Gitlab and use git@ for repo's
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/schedulers.py | 3 ++-
buildbot_gentoo_ci/steps/builders.py| 12
master.cfg | 12 ++--
3 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index 7c0aa6b..2e5379c 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -37,7 +37,8 @@ def getGitChanges(props):
change_data['committer'] = k['committer']
change_data['comments'] = k['comments']
change_data['revision'] = k['revision']
-change_data['timestamp'] =k['when_timestamp']
+change_data['timestamp'] = k['when_timestamp']
+change_data['branch'] = k['branch']
return change_data
def gentoo_schedulers():
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index ccb28e1..38dfa79 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -16,7 +16,7 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
from buildbot.process.results import WARNINGS
-from buildbot.plugins import steps
+from buildbot.plugins import steps, util
#FIXME: should be set in config
hosturl = 'http://90.231.13.235:8000'
@@ -374,7 +374,11 @@ class UpdateRepos(BuildStep):
mode='full',
submodules=True,
alwaysUseLatest=True,
-workdir=repository_path)
+workdir=repository_path,
+#FIXME: set filenames in repositorys db
+sshPrivateKey =
util.Secret("gitlab.gentoo.org_gentoo-ci.priv"),
+sshHostKey = util.Secret("gitlab.gentoo.org.host")
+)
])
return SUCCESS
@@ -1272,7 +1276,7 @@ class SetupStepts(BuildStep):
else:
for cpv, v in package_dict.items():
if re.search(cp, cpv):
-yield log.addStdout('Got' + cpv + '\n')
+yield log.addStdout('Got' + cpv + '\n')
yield log.addStdout('Match: NO\n')
# check for error
if stderr != []:
@@ -1284,7 +1288,7 @@ class SetupStepts(BuildStep):
yield log.addStdout('Error: NO\n')
return SKIPPED
build = True
-yield log.addStdout('Got' + self.getProperty("cpv") + '\n')
+yield log.addStdout('Got ' + self.getProperty("cpv") + '\n')
yield log.addStdout('Match: YES\n')
# update packages before any tests
if build:
diff --git a/master.cfg b/master.cfg
index 7d5af61..af50ec2 100644
--- a/master.cfg
+++ b/master.cfg
@@ -48,7 +48,7 @@ c['protocols'] = {'pb': {'port': 9989}}
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot version of a
python hello-world project.
-c['change_source'] = change_source.gentoo_change_source()
+#c['change_source'] = change_source.gentoo_change_source()
### SCHEDULERS
@@ -91,7 +91,15 @@ c['titleURL'] = "https://gentoo-ci.gentoo.org";
c['buildbotURL'] = "http://90.231.13.235:8010/";
# minimalistic config to activate new web UI
-c['www'] = dict(port=8010, plugins=dict(waterfall_view={}, console_view={},
grid_view={}))
+c['www'] = dict(
+port=8010,
+plugins=dict(waterfall_view={}, console_view={}, grid_view={}),
+change_hook_dialects={
+'gitlab' : {
+'secret': util.Secret("WWWHookGitlabToken"),
+},
+},
+)
c['www']['ui_default_config'] = {
'Waterfall.lazy_limit_waterfall': 408,
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/
commit: dae4e75d90b65bc77a383c63cd0d6aa19d832531
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Jul 14 23:43:18 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Jul 14 23:43:18 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=dae4e75d
Reado the match stage
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 27 +
buildbot_gentoo_ci/steps/builders.py | 166 ++---
2 files changed, 132 insertions(+), 61 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 426cee9..d07e31b 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -160,33 +160,10 @@ def run_build_request():
f.addStep(portage.SetPackageDefault())
# setup files in /etc if needed
# run --regen if needed on repo
-# update packages before any tests
-# run pretend on packages update on worker
-f.addStep(builders.RunEmerge(step='pre-update'))
-# look at the log to see if we need to do stuff
-# run update package on worker and check log
-f.addStep(builders.RunEmerge(step='update'))
-# clean up the worker
-# look at the log to see if we need to do stuff
-# run pre-depclean and depclean if set
-f.addStep(builders.RunEmerge(step='pre-depclean'))
-# run preserved-libs and depclean
-f.addStep(builders.RunEmerge(step='preserved-libs'))
-f.addStep(builders.RunEmerge(step='depclean'))
-# setup make.conf if build id has changes make.conf as dict from
SetMakeConf
-# setup package.* env if build id has changes
-# setup pkgcheck.conf if needed
-#f.addStep(builders.SetPkgCheckConf())
-# run pkgcheck if wanted
-# check log
-f.addStep(builders.RunPkgCheck())
# check cpv match
f.addStep(builders.RunEmerge(step='match'))
-# Add the needed steps for build
-f.addStep(builders.RunBuild())
-# run eclean pkg and dist
-#f.addStep(builders.RunEclean(step='pkg')
-#f.addStep(builders.RunEclean(step='dist')
+# Setup the needed stages for update, pkgcheck and build
+f.addStep(builders.SetupStepts())
return f
def parse_build_log():
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index dcd54c2..ccb28e1 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -15,6 +15,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
+from buildbot.process.results import WARNINGS
from buildbot.plugins import steps
#FIXME: should be set in config
@@ -26,9 +27,11 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['preserved_libs'] = False
emerge_output['change_use'] = False
emerge_output['circular_deps'] = False
+emerge_output['masked'] = False
emerge_output['failed'] = False
package_dict = {}
log_path_list = []
+print('stderr')
print(stderr)
# split the lines
for line in stdout.split('\n'):
@@ -115,7 +118,6 @@ def PersOutputOfEmerge(rc, stdout, stderr):
if change_use_list:
change_use[cpv_split[0]] = change_use_list
emerge_output['change_use'] = change_use
-err_line_list = []
if line.startswith(' * '):
if line.endswith('.log.gz'):
log_path = line.split(' ')[3]
@@ -125,6 +127,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
if line.endswith('circular dependencies:'):
emerge_output['circular_deps'] = True
stderr_line_list.append(line)
+if line.startswith('- '):
+if re.search('masked', line):
+stderr_line_list.append(line)
+emerge_output['masked'] = True
emerge_output['stderr'] = stderr_line_list
emerge_output['log_paths'] = log_path_list
@@ -397,6 +403,9 @@ class RunEmerge(BuildStep):
'emerge',
'-v'
]
+c = yield catpkgsplit(self.getProperty("cpv"))[0]
+p = yield catpkgsplit(self.getProperty("cpv"))[1]
+cp = c + '/' + p
aftersteps_list = []
#FIXME: Set build timeout in config
self.build_timeout = 6600
@@ -510,25 +519,10 @@ class RunEmerge(BuildStep):
aftersteps_list.append(CheckDepcleanLogs('depclean'))
if self.step == 'match':
-packages_excludes = yield
self.gentooci.db.projects.getProjectPortagePackageByUuidAndExclude(self.getProperty('project_data')['uuid'])
-cpv = self.getProperty("cpv")
-c = yield catpkgsplit(cpv)[0]
-p = yield catpkgsplit(cpv)[1]
-# Check if package is on the exclude list
-if packa
[gentoo-commits] proj/tinderbox-cluster:master commit in: docker/
commit: 51ab5319f6134dd57b74e4cc7a94caacd9941e83
Author: Magnus Granberg gentoo org>
AuthorDate: Thu Jul 14 00:29:16 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Thu Jul 14 00:29:16 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=51ab5319
Add docker file for log docker image
Signed-off-by: Magnus Granberg gentoo.org>
docker/GentooBuildbotWorkerDefault.Dockerfile | 26 ++
1 file changed, 26 insertions(+)
diff --git a/docker/GentooBuildbotWorkerDefault.Dockerfile
b/docker/GentooBuildbotWorkerDefault.Dockerfile
new file mode 100644
index 000..7e1a63b
--- /dev/null
+++ b/docker/GentooBuildbotWorkerDefault.Dockerfile
@@ -0,0 +1,26 @@
+# name the portage image
+FROM gentoo/portage:latest as portage
+
+# based on stage3 image
+FROM gentoo/stage3:latest
+
+# copy the entire portage volume in
+COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo
+
+# Setup portage
+# emerge needed deps buildbot-worker, psycopg and sqlalchemy
+# get the needed buildbot-worker config
+RUN echo -e "[binhost]\npriority = \nsync-uri =
https://gentoo.osuosl.org/experimental/amd64/binpkg/default/linux/17.1/x86-64/\n";
| cat >> /etc/portage/binrepos.conf\
+ && echo 'EMERGE_DEFAULT_OPTS="--binpkg-respect-use=n --usepkg=y --getbinpkg=y
--autounmask-write --autounmask-continue --autounmask-keep-keywords=y
--autounmask-use=y"' | cat >> /etc/portage/make.conf\
+ && echo 'FEATURES="-ipc-sandbox -pid-sandbox -network-sandbox -usersandbox
-mount-sandbox -sandbox"' | cat >> /etc/portage/make.conf\
+ && echo 'FEATURES="${FEATURES} parallel-install parallel-fetch -merge-sync"'
| cat >> /etc/portage/make.conf\
+ && echo 'FEATURES="${FEATURES} buildpkg"' | cat >> /etc/portage/make.conf\
+ && echo 'MAKEOPTS="-j8"' | cat >> /etc/portage/make.conf\
+ && echo 'dev-util/buildbot-worker' | cat >>
/etc/portage/package.accept_keywords/buildbot\
+ && emerge -qv buildbot-worker sqlalchemy dev-python/psycopg rust-bin\
+ && wget
https://raw.githubusercontent.com/buildbot/buildbot/master/worker/docker/buildbot.tac\
+ && cp buildbot.tac /var/lib/buildbot_worker/buildbot.tac
+
+WORKDIR /var/lib/buildbot_worker
+ENTRYPOINT ["/usr/bin/buildbot-worker"]
+CMD ["start", "--nodaemon"]
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 8d734b15cfcdc72915a0f2c63f186f4a5a4073b6
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 13 20:01:00 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 13 20:01:00 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8d734b15
Don't remove the repo dir
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/nodes.py
b/buildbot_gentoo_ci/steps/nodes.py
index 2a25562..a77c878 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -172,12 +172,12 @@ class SetupStage4Steps(BuildStep):
workdir=self.getProperty("workerbase")
))
# remove the gentoo repo
-aftersteps_list.append(steps.ShellCommand(
-flunkOnFailure=True,
-name='Remove the repo dir',
-command=['rm', '-R', self.getProperty('project_uuid') +
'/var/db/repos/gentoo'],
-workdir=self.getProperty("workerbase")
-))
+#aftersteps_list.append(steps.ShellCommand(
+#flunkOnFailure=True,
+#name='Remove the repo dir',
+#command=['rm', '-R', self.getProperty('project_uuid') +
'/var/db/repos/gentoo'],
+#workdir=self.getProperty("workerbase")
+#))
# compress it
aftersteps_list.append(steps.ShellCommand(
flunkOnFailure=True,
[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/config/
commit: 45fe4420dd0bc2057e50ff2382c070c214cfd68a
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 13 19:41:44 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 13 19:41:44 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=45fe4420
Add support log docker worker
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/config/builders.py | 19
buildbot_gentoo_ci/config/workers.py | 89 +++
master.cfg| 1 +
3 files changed, 48 insertions(+), 61 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index 6f3d606..e168cf5 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -23,12 +23,9 @@ def CanWorkerBuildProject(builder, wfb, request):
def gentoo_builders(worker_data):
b = []
g_ci_w = gentoo_ci_workers(worker_data)
-LocalWorkers = g_ci_w.getLocalWorkersUuid()
-BuildWorkers = g_ci_w.getBuildWorkersUuid()
-NodeWorkers = g_ci_w.getNodeWorkersUuid()
b.append(util.BuilderConfig(
name='update_db_check',
-workername=LocalWorkers[0],
+workername=g_ci_w.getWorkersUuid('local')[0],
workerbuilddir='builds',
collapseRequests=False,
factory=buildfactorys.update_db_check()
@@ -36,7 +33,7 @@ def gentoo_builders(worker_data):
)
b.append(util.BuilderConfig(
name='update_repo_check',
-workername=LocalWorkers[1],
+workername=g_ci_w.getWorkersUuid('local')[1],
workerbuilddir='builds',
collapseRequests=True,
factory=buildfactorys.update_repo_check()
@@ -48,7 +45,7 @@ def gentoo_builders(worker_data):
# first LocalWorker need to be done before we can use mulitplay workers
(git pull)
b.append(util.BuilderConfig(
name='update_cpv_data',
-workernames=LocalWorkers,
+workernames=g_ci_w.getWorkersUuid('local'),
workerbuilddir='builds',
collapseRequests=False,
factory=buildfactorys.update_db_cp()
@@ -57,7 +54,7 @@ def gentoo_builders(worker_data):
# Use multiplay workers
b.append(util.BuilderConfig(
name='update_v_data',
-workernames=LocalWorkers,
+workernames=g_ci_w.getWorkersUuid('local'),
workerbuilddir='builds',
collapseRequests=False,
factory=buildfactorys.update_db_v()
@@ -66,7 +63,7 @@ def gentoo_builders(worker_data):
# Use multiplay workers
b.append(util.BuilderConfig(
name='build_request_data',
-workernames=LocalWorkers,
+workernames=g_ci_w.getWorkersUuid('local'),
collapseRequests=False,
factory=buildfactorys.build_request_check()
)
@@ -74,7 +71,7 @@ def gentoo_builders(worker_data):
# Use multiplay workers
b.append(util.BuilderConfig(
name='run_build_request',
-workernames=BuildWorkers,
+workernames=g_ci_w.getWorkersUuid('docker'),
canStartBuild=CanWorkerBuildProject,
collapseRequests=False,
factory=buildfactorys.run_build_request()
@@ -83,7 +80,7 @@ def gentoo_builders(worker_data):
# Use multiplay workers
b.append(util.BuilderConfig(
name='parse_build_log',
-workernames=LocalWorkers,
+workernames=g_ci_w.getWorkersUuid('log'),
collapseRequests=False,
factory=buildfactorys.parse_build_log()
)
@@ -91,7 +88,7 @@ def gentoo_builders(worker_data):
# For node workers
b.append(util.BuilderConfig(
name='run_build_stage4_request',
-workernames=NodeWorkers,
+workernames=g_ci_w.getWorkersUuid('node'),
#FIXME: support more the one node
#canStartBuild=CanWorkerBuildProject,
collapseRequests=False,
diff --git a/buildbot_gentoo_ci/config/workers.py
b/buildbot_gentoo_ci/config/workers.py
index d4c20f7..42c6631 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -7,50 +7,30 @@ class gentoo_ci_workers():
def __init__(self, worker_data, **kwargs):
self.worker_data = worker_data
-def getLocalWorkersUuid(self):
-local_worker = []
+def getWorkersUuid(self, worker_type):
+worker_list = []
for worker in self.worker_data:
-if worker['type'] == 'local' and worker['enable'] is True:
-local_worker.append(worker['uuid'])
-print(local_worker)
-return local_worker
+if worker['type'] == worker_type and worker['enable'] is True:
+worker_list.append(worker['uuid'])
+print(worker_list)
+return worker_list
-def getBuildWorkersUuid(self):
-build_worker = []
+def getWorkersAllData(self, worker_type):
+worker_list = []
for worker in self.worker_data:
-if (worker['type'] != 'local'
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/
commit: bb330a4f317e06324138a80a4099cface0386b34 Author: Magnus Granberg gentoo org> AuthorDate: Wed Jul 13 19:50:55 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Wed Jul 13 19:50:55 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=bb330a4f Add ignore lines >>> / and ./usr/ Signed-off-by: Magnus Granberg gentoo.org> sql/search_pattern.sql | 8 +++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sql/search_pattern.sql b/sql/search_pattern.sql index d2e2197..50fb4e1 100644 --- a/sql/search_pattern.sql +++ b/sql/search_pattern.sql @@ -459,6 +459,7 @@ INSERT INTO public.projects_pattern VALUES (482, 'e89c2c1a-46e0-4ded-81dd-c51afe INSERT INTO public.projects_pattern VALUES (483, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', '^fatal: not a git repository', 0, 0, 'configure', 'info', 'search'); INSERT INTO public.projects_pattern VALUES (484, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', ', not found$', 0, 0, 'configure', 'info', 'search'); INSERT INTO public.projects_pattern VALUES (485, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'Could not find', 0, 0, 'configure', 'error', 'search'); +INSERT INTO public.projects_pattern VALUES (486, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'No such file or directory', 0, 0, 'configure', 'error', 'search'); -- File CATCH.install INSERT INTO public.projects_pattern VALUES (500, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'Can''t locate Locale/gettext.pm in @INC', 0, 0, 'install', 'error', 'search'); @@ -742,7 +743,12 @@ INSERT INTO public.projects_pattern VALUES (915, 'e89c2c1a-46e0-4ded-81dd-c51afe INSERT INTO public.projects_pattern VALUES (916, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'warning: python_fix_shebang', 0, 0, 'qa', 'warning', 'search'); INSERT INTO public.projects_pattern VALUES (917, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'SetuptoolsDeprecationWarning: 2to3', 0, 0, 'qa', 'warning', 'search'); INSERT INTO public.projects_pattern VALUES (918, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'SetuptoolsDeprecationWarning: setup.py install', 0, 0, 'qa', 'warning', 'search'); -INSERT INTO public.projects_pattern VALUES (919, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', ' * QA Notice:', 0, 0, 'qa', 'warning', 'startswith'); +INSERT INTO public.projects_pattern VALUES (919, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', '^ * QA Notice:', 0, 0, 'qa', 'warning', 'search'); + +-- ignore +INSERT INTO public.projects_pattern VALUES (950, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', '^>>> /', 0, 0, 'ignore', 'ignore', 'search'); +INSERT INTO public.projects_pattern VALUES (950, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', '^./usr/', 0, 0, 'ignore', 'ignore', 'search'); + -- -- TOC entry 2332 (class 0 OID 0)
[gentoo-commits] proj/tinderbox-cluster:master commit in: py/, bin/, buildbot_gentoo_ci/logs/, buildbot_gentoo_ci/steps/
commit: 1250899a4c17be77c99048576090ae1b32e6d227
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 13 19:49:23 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 13 19:49:23 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1250899a
Use log docker for log parser
Signed-off-by: Magnus Granberg gentoo.org>
bin/ci_log_parser | 20
buildbot_gentoo_ci/logs/log_parser.py | 180 --
buildbot_gentoo_ci/steps/logs.py | 88 ++---
py/log_parser.py | 159 ++
4 files changed, 212 insertions(+), 235 deletions(-)
diff --git a/bin/ci_log_parser b/bin/ci_log_parser
deleted file mode 100644
index 6401a49..000
--- a/bin/ci_log_parser
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2021 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-import argparse
-import sys
-from buildbot_gentoo_ci.logs.log_parser import runLogParser
-
-def main():
-# get filename, project_uuid default_project_uuid
-parser = argparse.ArgumentParser()
-parser.add_argument("-f", "--file", required=True)
-parser.add_argument("-u", "--uuid", required=True)
-args = parser.parse_args()
-runLogParser(args)
-sys.exit()
-
-if __name__ == "__main__":
-main()
diff --git a/buildbot_gentoo_ci/logs/log_parser.py
b/buildbot_gentoo_ci/logs/log_parser.py
deleted file mode 100644
index b890c12..000
--- a/buildbot_gentoo_ci/logs/log_parser.py
+++ /dev/null
@@ -1,180 +0,0 @@
-# Copyright 2021 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-import sys
-from multiprocessing import Pool, cpu_count
-import re
-import io
-import gzip
-import json
-import os
-from sqlalchemy.ext.declarative import declarative_base
-import sqlalchemy as sa
-
-Base = declarative_base()
-
-class ProjectsPattern(Base):
-__tablename__ = "projects_pattern"
-id = sa.Column(sa.Integer, primary_key=True)
-project_uuid = sa.Column(sa.String(36), nullable=False)
-search = sa.Column(sa.String(50), nullable=False)
-start = sa.Column(sa.Integer, default=0)
-end = sa.Column(sa.Integer, default=0)
-status = sa.Column(sa.Enum('info', 'warning', 'ignore', 'error'),
default='info')
-type = sa.Column(sa.Enum('info', 'qa', 'compile', 'configure', 'install',
'postinst', 'prepare', 'pretend', 'setup', 'test', 'unpack', 'ignore',
'issues', 'misc', 'elog'), default='info')
-search_type = sa.Column(sa.Enum('in', 'startswith', 'endswith', 'search'),
default='in')
-
-def getDBSession(config):
-#FIXME: Read the user/pass from file
-engine = sa.create_engine(config['database'])
-Session = sa.orm.sessionmaker(bind = engine)
-return Session()
-
-def getMultiprocessingPool(config):
-return Pool(processes = int(config['core']))
-
-def addPatternToList(Session, pattern_list, uuid):
-for project_pattern in
Session.query(ProjectsPattern).filter_by(project_uuid=uuid).all():
-# check if the search pattern is vaild
-project_pattern_search = project_pattern.search
-try:
-re.compile(project_pattern_search)
-except re.error:
-print("Non valid regex pattern")
-print(project_pattern.search)
-print(project_pattern.id)
-else:
-patten_dict = {}
-patten_dict['id'] = project_pattern.id
-patten_dict['project_uuid'] = project_pattern.project_uuid
-patten_dict['search'] = project_pattern_search
-patten_dict['start'] = project_pattern.start
-patten_dict['end'] = project_pattern.end
-patten_dict['status'] = project_pattern.status
-patten_dict['type'] = project_pattern.type
-patten_dict['search_type'] = project_pattern.search_type
-pattern_list.append(patten_dict)
-return pattern_list
-
-def get_log_search_pattern(Session, uuid, default_uuid):
-# get pattern from the projects
-# add that to log_search_pattern_list
-log_search_pattern_list = []
-log_search_pattern_list = addPatternToList(Session,
log_search_pattern_list, uuid)
-log_search_pattern_list = addPatternToList(Session,
log_search_pattern_list, default_uuid)
-return log_search_pattern_list
-
-def search_buildlog(log_search_pattern_list, logfile_text_dict, tmp_index,
max_text_lines):
-# get text line to search
-text_line = logfile_text_dict[tmp_index]
-summery_dict = {}
-# loop true the pattern list for match
-for search_pattern in log_search_pattern_list:
-search_hit = False
-ignore_line = False
-# check if should ignore the line
-#FIXME take the ignore line pattern from db
-if re.search('^>>> /', text_line):
-ignore_line = True
-#if else re.search('./\w+/'):
-#pass
-
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/
commit: e6adf0b1f7cad72329eed2d90021e024d43b1f83 Author: Magnus Granberg gentoo org> AuthorDate: Wed Jul 13 19:53:13 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Wed Jul 13 19:53:13 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e6adf0b1 Use ReadEmergeInfoLog Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/buildfactorys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index ef68f2d..426cee9 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -207,7 +207,7 @@ def parse_build_log(): # add sum log to buildbot log f.addStep(logs.setBuildbotLog()) # pers the emerge info -f.addStep(logs.SetupParserEmergeInfoLog()) +f.addStep(logs.ReadEmergeInfoLog()) # add emerge info to log and db f.addStep(logs.setEmergeInfoLog()) # add package info to log and db
[gentoo-commits] proj/tinderbox-cluster:master commit in: sql/
commit: 43f7d0d6fe1817b5b8c21beb11207f4f9fa17324 Author: Magnus Granberg gentoo org> AuthorDate: Fri Jul 8 21:56:07 2022 + Commit: Magnus Granberg gentoo org> CommitDate: Fri Jul 8 21:56:07 2022 + URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=43f7d0d6 Add 2 configure search pattern Signed-off-by: Magnus Granberg gentoo.org> sql/search_pattern.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/search_pattern.sql b/sql/search_pattern.sql index 31d4692..d2e2197 100644 --- a/sql/search_pattern.sql +++ b/sql/search_pattern.sql @@ -457,6 +457,8 @@ INSERT INTO public.projects_pattern VALUES (480, 'e89c2c1a-46e0-4ded-81dd-c51afe INSERT INTO public.projects_pattern VALUES (481, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', ' \* Failed to find configuration', 0, 0, 'configure', 'error', 'search'); INSERT INTO public.projects_pattern VALUES (482, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'Reason: UndefinedError', 0, 0, 'configure', 'error', 'search'); INSERT INTO public.projects_pattern VALUES (483, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', '^fatal: not a git repository', 0, 0, 'configure', 'info', 'search'); +INSERT INTO public.projects_pattern VALUES (484, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', ', not found$', 0, 0, 'configure', 'info', 'search'); +INSERT INTO public.projects_pattern VALUES (485, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'Could not find', 0, 0, 'configure', 'error', 'search'); -- File CATCH.install INSERT INTO public.projects_pattern VALUES (500, 'e89c2c1a-46e0-4ded-81dd-c51afeb7fcff', 'Can''t locate Locale/gettext.pm in @INC', 0, 0, 'install', 'error', 'search');
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 26b8a9034b0ce05535fb8730a4a43dcf4f70aa14
Author: Magnus Granberg gentoo org>
AuthorDate: Fri Jul 8 21:53:50 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Fri Jul 8 21:53:50 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=26b8a903
Fix a typo for Title
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index e5e61a5..6bd3279 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -301,7 +301,7 @@ class MakeIssue(BuildStep):
else:
cpv = self.getProperty('cpv')
self.error_dict['cpv'] = cpv
-yield log.addStdout('Titel:' + '\n')
+yield log.addStdout('Title:' + '\n')
yield log.addStdout(separator2.join([cpv, '-',
self.error_dict['title']]) + separator1)
yield log.addStdout('Summary:' + '\n')
for line in self.summary_log_list:
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 86dae64f04e20bf74405496289564cd993429c03
Author: Magnus Granberg gentoo org>
AuthorDate: Wed Jul 6 18:44:08 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Wed Jul 6 18:44:08 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=86dae64f
Change title_issue and title_phase
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 2 +-
buildbot_gentoo_ci/steps/logs.py | 29 +++--
2 files changed, 4 insertions(+), 27 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index e43c926..27b1388 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -88,7 +88,7 @@ class GetBugs(BuildStep):
match = False
for bug in buglist:
yield log.addStdout('Bug: ' + str(bug['id']) + ' Summary: ' +
bug['summary'] +'\n')
-if re.search(self.getProperty('error_dict')['title_phase'][:20],
bug['summary']):
+if re.search(self.getProperty('error_dict')['title_issue'][:20],
bug['summary']):
print('Bug found')
print(bug)
match = {}
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 7a63658..e5e61a5 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -318,39 +318,16 @@ class MakeIssue(BuildStep):
text_phase_list = []
for k, v in sorted(self.summary_log_dict.items()):
# get the issue error
-if v['text'].startswith(' * ERROR:') and v['text'].endswith('
phase):'):
-issue_text = self.summary_log_dict[k + 1]['text']
-if issue_text.startswith(' * ninja -v -j'):
-issue_text = 'ninja failed'
-if issue_text.startswith(' * (no error'):
-issue_text = False
-if issue_text:
-text_issue_list.append(issue_text)
-# get the phase error
if v['type'] == self.error_dict['phase'] and v['status'] ==
'error':
-text_phase_list.append(v['text'])
-# if not get the first issue
-if text_issue_list == []:
-for k, v in self.summary_log_dict.items():
-if v['type'] == 'issues':
-text_issue_list.append(v['text'])
-# if not get the first error
-if text_phase_list == []:
-for k, v in self.summary_log_dict.items():
-if v['status'] == 'error':
-text_phase_list.append(v['text'])
+text_issue_list.append(v['text'])
# add the issue error
if text_issue_list != []:
self.error_dict['title_issue'] = text_issue_list[0].replace('*',
'').strip()
else:
self.error_dict['title_issue'] = 'title_issue : None'
-# add the error line
-if text_phase_list != []:
-self.error_dict['title_phase'] = text_phase_list[0].replace('*',
'').strip()
-else:
-self.error_dict['title_phase'] = 'title_phase : None'
+self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase']
#set the error title
-self.error_dict['title'] = self.error_dict['title_issue'] + ' (' +
self.error_dict['title_phase'] + ')'
+self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' +
self.error_dict['title_issue']
@defer.inlineCallbacks
def run(self):
[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
commit: 0486f29cc0842870d85029553f2c229799896fa5
Author: Magnus Granberg gentoo org>
AuthorDate: Mon Jul 4 17:43:38 2022 +
Commit: Magnus Granberg gentoo org>
CommitDate: Mon Jul 4 17:43:38 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0486f29c
Add support for search bugs on b.g.o
Signed-off-by: Magnus Granberg gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 124 +++
buildbot_gentoo_ci/steps/logs.py | 14 -
2 files changed, 135 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
new file mode 100644
index 000..e43c926
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -0,0 +1,124 @@
+# Copyright 2022 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import re
+
+from twisted.internet import defer
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
+
+from bugz.cli import check_bugz_token, login, list_bugs
+from bugz.cli_argparser import make_arg_parser
+from bugz.configfile import load_config
+from bugz.settings import Settings
+from bugz.exceptions import BugzError
+from bugz.log import log_error, log_info
+
+from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
+
+# Origins: bugz.cli
+# Modifyed by Gentoo Authors.
+# main
+def main_bugz(args):
+ArgParser = make_arg_parser()
+opt = ArgParser.parse_args(args)
+
+ConfigParser = load_config(getattr(opt, 'config_file', None))
+
+check_bugz_token()
+settings = Settings(opt, ConfigParser)
+return settings
+
+# search
+def search_bugz(args):
+settings = main_bugz(args)
+valid_keys = ['alias', 'assigned_to', 'component', 'creator',
+'limit', 'offset', 'op_sys', 'platform',
+'priority', 'product', 'resolution', 'severity',
+'version', 'whiteboard', 'cc']
+
+params = {}
+d = vars(settings)
+for key in d:
+if key in valid_keys:
+params[key] = d[key]
+if 'search_statuses' in d:
+if 'all' not in d['search_statuses']:
+params['status'] = d['search_statuses']
+if 'terms' in d:
+params['summary'] = d['terms']
+
+if not params:
+raise BugzError('Please give search terms or options.')
+
+log_info('Searching for bugs meeting the following criteria:')
+for key in params:
+log_info(' {0:<20} = {1}'.format(key, params[key]))
+
+login(settings)
+
+result = settings.call_bz(settings.bz.Bug.search, params)['bugs']
+
+if not len(result):
+log_info('No bugs found.')
+return []
+else:
+list_bugs(result, settings)
+return result
+
+class GetBugs(BuildStep):
+
+name = 'GetBugs'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def find_match(self, buglist):
+log = yield self.addLog('Bugs')
+yield log.addStdout('Open Bugs\n')
+match = False
+for bug in buglist:
+yield log.addStdout('Bug: ' + str(bug['id']) + ' Summary: ' +
bug['summary'] +'\n')
+if re.search(self.getProperty('error_dict')['title_phase'][:20],
bug['summary']):
+print('Bug found')
+print(bug)
+match = {}
+match['id'] = bug['id']
+match['summary'] = bug['summary']
+if match:
+yield log.addStdout('Match bug found\n')
+yield log.addStdout('Bug: ' + str(match['id']) + ' Summary: ' +
match['summary'] +'\n')
+self.setProperty("bgo", match, 'bgo')
+return
+yield log.addStdout('NO Match bug found\n')
+self.setProperty("bgo", False, 'bgo')
+
+@defer.inlineCallbacks
+def run(self):
+# self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+cpv = self.getProperty('error_dict')['cpv']
+c = yield catpkgsplit(cpv)[0]
+p = yield catpkgsplit(cpv)[1]
+cp = c + '/' + p
+# search for open bugs
+args = []
+args.append('--skip-auth')
+args.append('search')
+# set limit
+# set date last 30 days
+# search for cp
+args.append(cp)
+print(args)
+buglist = search_bugz(args)
+print(buglist)
+self.find_match(buglist)
+return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index d6ee705..7a63658 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -22,6 +22,7 @@ from buildbot.plugins import steps
from buildb
