[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, sql/, ...

2024-02-29 Thread Magnus Granberg 
commit: 4269430378d05d5161d87056caf6817890d44493
Author: Magnus Granberg  gentoo  org>
AuthorDate: Thu Feb 29 20:52:42 2024 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Thu Feb 29 20:52:42 2024 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=42694303

Make emails dict for bugreports

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/schedulers.py |  5 +++--
 buildbot_gentoo_ci/db/model.py  |  1 +
 buildbot_gentoo_ci/db/versions.py   |  6 --
 buildbot_gentoo_ci/steps/logs.py| 26 +++---
 buildbot_gentoo_ci/steps/version.py |  4 +++-
 sql/gentoo_ci_schema.sql|  1 +
 6 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index c34d714..a4180cc 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -40,19 +40,20 @@ def getGitChanges(props):
 change_data['timestamp'] = k['when_timestamp']
 change_data['branch'] = k['branch']
 change_data['project'] = k['project']
+change_data['change_id'] = k['changeid']
 return change_data
 
 def bb_branch_fn(branch):
 # check branch
 print(f"Branch: {branch}")
-if branch.endswith('-mr') or branch.endswith('-pr') or branch == 'master':
+if branch.endswith('-mr') or branch.endswith('-pr') or 
branch.endswith('master') or branch.endswith('main'):
 return True
 return False
 
 def bb_category_fn(category):
 # check event
 print(f"Category: {category}")
-if category == 'push' or category == 'merge_request':
+if category == 'push' or category == 'merge_request' or category is None:
 return True
 return False
 

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index b6a0c43..dea3e15 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -348,6 +348,7 @@ class Model(base.DBConnectorComponent):
   nullable=False),
 sa.Column('file_hash', sa.String(255), nullable=False),
 sa.Column('commit_id', sa.String(255), nullable=False),
+sa.Column('change_id', sa.Integer, nullable=True, default=0),
 sa.Column('deleted', sa.Boolean, default=False),
 sa.Column('deleted_at', sa.Integer, nullable=True),
 )

diff --git a/buildbot_gentoo_ci/db/versions.py 
b/buildbot_gentoo_ci/db/versions.py
index 0aaac1b..8d35602 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -55,7 +55,7 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
 return res
 
 @defer.inlineCallbacks
-def addVersion(self, name, package_uuid, file_hash, commit_id):
+def addVersion(self, name, package_uuid, file_hash, commit_id, change_id):
 def thd(conn, no_recurse=False):
 try:
 tbl = self.db.model.versions
@@ -63,7 +63,8 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
 r = conn.execute(q, dict(name=name,
  package_uuid=package_uuid,
  file_hash=file_hash,
- commit_id=commit_id))
+ commit_id=commit_id,
+ change_id=change_id))
 except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
 uuid = None
 else:
@@ -178,6 +179,7 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
 package_uuid=row.package_uuid,
 file_hash=row.file_hash,
 commit_id=row.commit_id,
+change_id = row.change_id,
 deleted=row.deleted,
 deleted_at=row.deleted_at
 )

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index aacb8a5..34aee29 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -250,6 +250,7 @@ class MakeIssue(BuildStep):
 self.summary_log_list = []
 self.error_dict = {}
 self.aftersteps_list = []
+Maintainers = []
 #self.error_dict['hash'] = hashlib.sha256()
 for k, v in sorted(self.summary_log_dict.items()):
 self.summary_log_list.append(v['text'])
@@ -263,11 +264,30 @@ class MakeIssue(BuildStep):
 self.error_dict['phase'] = v['text'].split(' (')[1].split(' 
phase')[0]
 error = True
 if v['text'].startswith(' * Maintainer:'):
-Maintainers = []
 for email in v['text'].split(':')[1].split(' '):
 if email != '':
-Maintainers.append(email)
-self.setProperty("Maintainers", Maintainers, 'Maintainers')
+for e in email.split(','):
+

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2022-11-09 Thread Magnus Granberg 
commit: ea8cb4a47f01964a09a7bd6f863d871a01c8b76d
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Nov  9 21:05:15 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Nov  9 21:05:15 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ea8cb4a4

Rename setBuildbotLog to Summarylog

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 2 +-
 buildbot_gentoo_ci/steps/logs.py   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 7f2dff9..d07a143 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -180,7 +180,7 @@ def parse_build_log():
 # set it SUCCESS/FAILURE/WARNINGS
 f.addStep(logs.MakeIssue())
 # add sum log to buildbot log
-f.addStep(logs.setBuildbotLog())
+f.addStep(logs.Summarylog())
 # pers the emerge info
 f.addStep(logs.ReadEmergeInfoLog())
 # add emerge info to log and db

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 67845e7..2a52308 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -404,9 +404,9 @@ class MakeIssue(BuildStep):
 yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
 return SUCCESS
 
-class setBuildbotLog(BuildStep):
+class Summarylog(BuildStep):
 
-name = 'setBuildbotLog'
+name = 'Summarylog'
 description = 'Running'
 descriptionDone = 'Ran'
 descriptionSuffix = None

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2022-09-01 Thread Magnus Granberg 
commit: 0d19b4627eba83ffccadc753105b5cbdf53a68e7
Author: Magnus Granberg  gentoo  org>
AuthorDate: Thu Sep  1 14:56:42 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Thu Sep  1 14:56:42 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0d19b462

Add Bug ID for Irc reporter and nice titel

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/reporters.py |  7 ---
 buildbot_gentoo_ci/steps/bugs.py   | 24 
 buildbot_gentoo_ci/steps/logs.py   | 16 
 3 files changed, 36 insertions(+), 11 deletions(-)

diff --git a/buildbot_gentoo_ci/config/reporters.py 
b/buildbot_gentoo_ci/config/reporters.py
index 9832305..bd55037 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -8,10 +8,11 @@ from buildbot.reporters.message import MessageFormatter
 
 from buildbot_gentoo_ci.reporters import irc
 irc_template = '''{% set resultsList = ["\x0303SUCCESS", "\x0308WARNINGS", 
"\x0304FAILURE"] %}\
-{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303" 
}}repo/{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
+{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303" 
}}Repo:{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
 {{ build['properties']['revision'][0]|truncate(10, True) }} {{ "\x0302" }}{{ 
build['properties']['owners'][0][0] }}{{ "\x03" }} \
-{{ build['properties']['event'][0] }} {{ projects }}:{{ 
build['properties']['project_data'][0]['name'] }} \
-{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{ 
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }}\
+{{ "\x0306" }}{{ build['properties']['event'][0] }}{{ "\x03" }} {{ projects 
}}:{{ build['properties']['project_data'][0]['name'] }} \
+{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{ 
"\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }} \
+{% if build['properties']['bgo'][0]['match'] is true %}{{ "\x0311" }}Bugid: 
{{build['properties']['bgo'][0]['id']}}{{ "\x03" }}{% endif %}\
 '''
 
 def ircGenerators():

diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 27b1388..801fc98 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -87,20 +87,28 @@ class GetBugs(BuildStep):
 yield log.addStdout('Open Bugs\n')
 match = False
 for bug in buglist:
-yield log.addStdout('Bug: ' + str(bug['id']) + ' Summary: ' + 
bug['summary'] +'\n')
-if re.search(self.getProperty('error_dict')['title_issue'][:20], 
bug['summary']):
-print('Bug found')
-print(bug)
+yield log.addStdout(f"Bug: {str(bug['id'])} Summary: 
{bug['summary']}\n")
+# we splite the lines to lists and try to match the words
+matches = 0
+match_search_text = 
list(self.getProperty('error_dict')['title_issue'].split())
+match_bug_text = list(bug['summary'].split())
+#FIXME: add check for cp
+for match_word in match_search_text:
+if match_word in match_bug_text:
+matches = matches + 1
+if matches >= 10:
+print(f"Bug: {str(bug['id'])} Summary: {bug['summary']}")
 match = {}
+match['match'] = True
 match['id'] = bug['id']
 match['summary'] = bug['summary']
+yield log.addStdout(f"Line to match: 
{self.getProperty('error_dict')['title_issue']}\n")
 if match:
-yield log.addStdout('Match bug found\n')
-yield log.addStdout('Bug: ' + str(match['id']) + ' Summary: ' + 
match['summary'] +'\n')
+yield log.addStdout('Match bug: YES\n')
+yield log.addStdout(f"Bug: {str(match['id'])} Summary: 
{match['summary']}\n")
 self.setProperty("bgo", match, 'bgo')
 return
-yield log.addStdout('NO Match bug found\n')
-self.setProperty("bgo", False, 'bgo')
+yield log.addStdout('Match bug: NO\n')
 
 @defer.inlineCallbacks
 def run(self):

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 164bb24..57f9394 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -91,6 +91,7 @@ class SetupPropertys(BuildStep):
 else:
 log_cpv = self.getProperty('cpv')
 self.setProperty("log_cpv", log_cpv, 'log_cpv')
+self.setProperty("bgo", dict( match=False), 'bgo')
 self.descriptionDone = 'Runing log checker on ' + log_cpv
 return SUCCESS
 
@@ -333,6 +334,19 @@ class MakeIssue(BuildStep):
 yield log.addStdout(log_cpv['full_logname'] + '\n')
 yield log.addStdout('world.log' + '\n')
 
+def getNiceErrorLine(self, line):
+# strip away hex addresses, loong p

[gentoo-commits] proj/tinderbox-cluster:master commit in: /, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2022-07-22 Thread Magnus Granberg 
commit: 07a0c6eb02b632e88bc8d288c33015c8b43e93e0
Author: Magnus Granberg  gentoo  org>
AuthorDate: Fri Jul 22 22:18:44 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Fri Jul 22 22:18:44 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=07a0c6eb

Use Webhook for Gitlab and use git@ for repo's

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/schedulers.py |  3 ++-
 buildbot_gentoo_ci/steps/builders.py| 12 
 master.cfg  | 12 ++--
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index 7c0aa6b..2e5379c 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -37,7 +37,8 @@ def getGitChanges(props):
 change_data['committer'] = k['committer']
 change_data['comments'] = k['comments']
 change_data['revision'] = k['revision']
-change_data['timestamp'] =k['when_timestamp']
+change_data['timestamp'] = k['when_timestamp']
+change_data['branch'] = k['branch']
 return change_data
 
 def gentoo_schedulers():

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index ccb28e1..38dfa79 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -16,7 +16,7 @@ from buildbot.process.results import SUCCESS
 from buildbot.process.results import FAILURE
 from buildbot.process.results import SKIPPED
 from buildbot.process.results import WARNINGS
-from buildbot.plugins import steps
+from buildbot.plugins import steps, util
 
 #FIXME: should be set in config
 hosturl = 'http://90.231.13.235:8000'
@@ -374,7 +374,11 @@ class UpdateRepos(BuildStep):
 mode='full',
 submodules=True,
 alwaysUseLatest=True,
-workdir=repository_path)
+workdir=repository_path,
+#FIXME: set filenames in repositorys db
+sshPrivateKey = 
util.Secret("gitlab.gentoo.org_gentoo-ci.priv"),
+sshHostKey = util.Secret("gitlab.gentoo.org.host")
+)
 ])
 return SUCCESS
 
@@ -1272,7 +1276,7 @@ class SetupStepts(BuildStep):
 else:
 for cpv, v in package_dict.items():
 if re.search(cp, cpv):
-yield log.addStdout('Got' + cpv + '\n')
+yield log.addStdout('Got'  + cpv + '\n')
 yield log.addStdout('Match: NO\n')
 # check for error
 if stderr != []:
@@ -1284,7 +1288,7 @@ class SetupStepts(BuildStep):
 yield log.addStdout('Error: NO\n')
 return SKIPPED
 build = True
-yield log.addStdout('Got' + self.getProperty("cpv") + '\n')
+yield log.addStdout('Got ' + self.getProperty("cpv") + '\n')
 yield log.addStdout('Match: YES\n')
 # update packages before any tests
 if build:

diff --git a/master.cfg b/master.cfg
index 7d5af61..af50ec2 100644
--- a/master.cfg
+++ b/master.cfg
@@ -48,7 +48,7 @@ c['protocols'] = {'pb': {'port': 9989}}
 # the 'change_source' setting tells the buildmaster how it should find out
 # about source code changes.  Here we point to the buildbot version of a 
python hello-world project.
 
-c['change_source'] = change_source.gentoo_change_source()
+#c['change_source'] = change_source.gentoo_change_source()
 
 ### SCHEDULERS
 
@@ -91,7 +91,15 @@ c['titleURL'] = "https://gentoo-ci.gentoo.org";
 c['buildbotURL'] = "http://90.231.13.235:8010/";
 
 # minimalistic config to activate new web UI
-c['www'] = dict(port=8010, plugins=dict(waterfall_view={}, console_view={}, 
grid_view={}))
+c['www'] = dict(
+port=8010,
+plugins=dict(waterfall_view={}, console_view={}, grid_view={}),
+change_hook_dialects={
+'gitlab' : {
+'secret': util.Secret("WWWHookGitlabToken"),
+},
+},
+)
 
 c['www']['ui_default_config'] = {
 'Waterfall.lazy_limit_waterfall': 408,

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2022-05-28 Thread Magnus Granberg 
commit: 1f57c498694cbdce80ad43b2b5a3c0c44cdee588
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sat May 28 14:54:50 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sat May 28 14:54:50 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1f57c498

Add support for docker latent workers

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/workers.py | 39 ++--
 buildbot_gentoo_ci/steps/builders.py | 15 --
 2 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/buildbot_gentoo_ci/config/workers.py 
b/buildbot_gentoo_ci/config/workers.py
index fbdc2a9..d4c20f7 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -1,7 +1,7 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2022 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-from buildbot.plugins import worker
+from buildbot.plugins import worker, util
 
 class gentoo_ci_workers():
 def __init__(self, worker_data, **kwargs):
@@ -47,17 +47,52 @@ class gentoo_ci_workers():
 print(node_worker)
 return node_worker
 
+@util.renderer
+def docker_images(props):
+return 'bb-worker-' + props.getProperty('project_uuid') + ':latest'
+
+@util.renderer
+def docker_volumes(props):
+volumes_list = []
+#FIXME: set in master.cfg
+src_dir = '/srv/gentoo/portage/' + props.getProperty('project_uuid')
+dest_dir = '/var/cache/portage'
+#add distdir
+volumes_list.append(src_dir + '/distfiles' + ':' + dest_dir + '/distfiles')
+#add bindir
+volumes_list.append(src_dir + '/packages' + ':' + dest_dir + '/packages')
+return volumes_list
+
 def gentoo_workers(worker_data):
 w = []
 g_ci_w = gentoo_ci_workers(worker_data)
 LocalWorkers = g_ci_w.getLocalWorkersUuid()
 BuildWorkers = g_ci_w.getBuildWorkersAllData()
 NodeWorkers = g_ci_w.getNodedWorkersAllData()
+docker_hostconfig = {}
+# For use of sandbox stuff
+# FEATURES="ipc-sandbox network-sandbox pid-sandbox"
+docker_hostconfig['cap_add'] = ['SYS_ADMIN', 'NET_ADMIN', 'SYS_PTRACE']
+# libseccomp overhead
+# https://github.com/seccomp/libseccomp/issues/153
+docker_hostconfig['security_opt'] = ['seccomp=unconfined']
 for local_worker in LocalWorkers:
 w.append(worker.LocalWorker(local_worker))
 for build_worker in BuildWorkers:
 if build_worker['type'] == 'default':
 w.append(worker.Worker(build_worker['uuid'], 
build_worker['password']))
+#FIXME: set settings in master.cfg
+if build_worker['type'] == 'docker':
+w.append(worker.DockerLatentWorker(build_worker['uuid'],
+build_worker['password'],
+docker_host='tcp://192.168.1.3:2375',
+image=docker_images,
+volumes=docker_volumes,
+hostconfig=docker_hostconfig,
+followStartupLogs=True,
+masterFQDN='192.168.1.5',
+build_wait_timeout=3600
+))
 for node_worker in NodeWorkers:
 if node_worker['type'] == 'node':
 w.append(worker.Worker(node_worker['uuid'], 
node_worker['password']))

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 8ce6545..1d4f52d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -236,7 +236,8 @@ class TriggerRunBuildRequest(BuildStep):
 'projectrepository_data' : 
self.projectrepository_data,
 'use_data' : self.use_data,
 'fullcheck' : self.getProperty("fullcheck"),
-'project_build_data' : project_build_data
+'project_build_data' : project_build_data,
+'project_uuid' : self.project_data['uuid']
 }
 )])
 return SUCCESS
@@ -384,6 +385,8 @@ class RunEmerge(BuildStep):
 self.descriptionSuffix = self.step
 self.name = 'Setup emerge for ' + self.step + ' step'
 self.build_env = {}
+#FIXME: Set build timeout in config
+self.build_timeout = 1800
 
 @defer.inlineCallbacks
 def run(self):
@@ -454,7 +457,7 @@ class RunEmerge(BuildStep):
 strip=True,
 extract_fn=PersOutputOfEmerge,
 workdir='/',
-timeout=None
+timeout=self.build_timeout
 ))
 aftersteps_list.append(CheckEmergeLogs('update'))
 if projects_emerge_options['preserved_libs']:
@@ -470,7 +473,7 @@ class RunEmerge(BuildStep):
 strip=True,

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/

2022-04-22 Thread Magnus Granberg 
commit: 2fac94e594b5f50b00780361788df91800a59ea7
Author: Magnus Granberg  gentoo  org>
AuthorDate: Fri Apr 22 12:37:35 2022 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Fri Apr 22 12:37:35 2022 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2fac94e5

Add support stage4 build

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |   9 +
 buildbot_gentoo_ci/config/schedulers.py|  28 ++-
 buildbot_gentoo_ci/db/model.py |   1 +
 buildbot_gentoo_ci/steps/nodes.py  | 322 +
 4 files changed, 344 insertions(+), 16 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 86fa162..a714e47 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -13,6 +13,7 @@ from buildbot_gentoo_ci.steps import builders
 from buildbot_gentoo_ci.steps import portage
 from buildbot_gentoo_ci.steps import logs
 from buildbot_gentoo_ci.steps import repos
+from buildbot_gentoo_ci.steps import nodes
 
 def update_db_check():
 f = util.BuildFactory()
@@ -216,3 +217,11 @@ def parse_build_log():
 # setup things for the irc bot
 #f.addStep(logs.SetIrcInfo())
 return f
+
+def run_build_stage4_request():
+f = util.BuildFactory()
+# set needed Propertys
+f.addStep(nodes.SetupPropertys())
+# set the needed steps for making the stage4
+f.addStep(nodes.SetupStage4Steps())
+return f

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index e520076..8c08006 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -46,24 +46,18 @@ def gentoo_schedulers():
 builderNames = builderUpdateDbNames,
 change_filter=util.ChangeFilter(branch='master'),
 )
-test_updatedb = schedulers.ForceScheduler(
-name="force",
-buttonName="pushMe!",
-label="My nice Force form",
-builderNames=['update_db_check'],
+create_stage4 = schedulers.ForceScheduler(
+name="create_stage4",
+buttonName="Create stage4",
+label="Create stage4 form",
+builderNames=['run_build_stage4_request'],
 # A completely customized property list.  The name of the
 # property is the name of the parameter
 properties=[
-util.NestedParameter(name="options", label="Build Options",
-layout="vertical", fields=[
-util.StringParameter(name="cpv_changes",
-label="Package to check",
-default="dev-lang/python-3.8", size=80),
-util.StringParameter(name="repository",
-label="repo",
-default="gentoo", size=80),
-])
-])
+util.StringParameter(name="project_uuid",
+label="Project uuid",
+default="e89c2c1a-46e0-4ded-81dd-c51afeb7fcfd", size=36),
+])
 update_cpv_data = schedulers.Triggerable(name="update_cpv_data",
builderNames=["update_cpv_data"])
 update_repo_check = schedulers.Triggerable(name="update_repo_check",
@@ -76,8 +70,10 @@ def gentoo_schedulers():
builderNames=["run_build_request"])
 parse_build_log = schedulers.Triggerable(name="parse_build_log",
builderNames=["parse_build_log"])
+run_build_stage4_request = 
schedulers.Triggerable(name="run_build_stage4_request",
+   builderNames=["run_build_stage4_request"])
 s = []
-s.append(test_updatedb)
+s.append(create_stage4)
 s.append(scheduler_update_db)
 s.append(update_repo_check)
 s.append(update_cpv_data)

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 18fb7ff..908ebb5 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -110,6 +110,7 @@ class Model(base.DBConnectorComponent):
 sa.Column('keyword_id', sa.Integer,
   sa.ForeignKey('keywords.id', ondelete='CASCADE'),
   nullable=False),
+sa.Column('image', sa.String(255), nullable=False),
 sa.Column('status', sa.Enum('stable','unstable','all'), 
nullable=False),
 sa.Column('auto', sa.Boolean, default=False),
 sa.Column('enabled', sa.Boolean, default=False),

diff --git a/buildbot_gentoo_ci/steps/nodes.py 
b/buildbot_gentoo_ci/steps/nodes.py
new file mode 100644
index 000..37cf917
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -0,0 +1,322 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import re
+import json
+import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
+

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2021-12-23 Thread Magnus Granberg 
commit: b3edf64aaff42659b3593dfe24a7e0a68e1881a0
Author: Magnus Granberg  gentoo  org>
AuthorDate: Fri Dec 24 00:33:19 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Fri Dec 24 00:33:19 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b3edf64a

put emerge and package info to a file insted

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  4 ++
 buildbot_gentoo_ci/steps/builders.py   | 74 +++--
 buildbot_gentoo_ci/steps/logs.py   | 75 +-
 3 files changed, 137 insertions(+), 16 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 27f0076..6203a1f 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -172,8 +172,12 @@ def parse_build_log():
 f.addStep(logs.MakeIssue())
 # add sum log to buildbot log
 f.addStep(logs.setBuildbotLog())
+# pers the emerge info
+f.addStep(logs.SetupParserEmergeInfoLog())
 # add emerge info to log and db
 f.addStep(logs.setEmergeInfoLog())
+# add package info to log and db
+f.addStep(logs.setPackageInfoLog())
 # set BuildStatus
 f.addStep(logs.setBuildStatus())
 # setup things for the irc bot

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 7247abd..e76fe3d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -590,14 +590,35 @@ class CheckEmergeLogs(BuildStep):
 workdir=workdir
 ))
 
-@defer.inlineCallbacks
-def getLogFile(self, cpv, log_dict):
-masterdest = yield os.path.join(self.master.basedir, 'workers', 
self.getProperty('workername'), str(self.getProperty("buildnumber")) 
,log_dict[cpv]['full_logname'])
+def addFileUploade(self, sourcefile, destfile):
 self.aftersteps_list.append(steps.FileUpload(
-workersrc=log_dict[cpv]['log_path'],
-masterdest=masterdest
+workersrc=sourcefile,
+masterdest=destfile
 ))
 
+@defer.inlineCallbacks
+def getLogFile(self, cpv, log_dict):
+destfile = yield os.path.join(self.masterdest, 
log_dict[cpv]['full_logname'])
+sourcefile = log_dict[cpv]['log_path']
+self.addFileUploade(sourcefile, destfile)
+
+@defer.inlineCallbacks
+def getEmergeFiles(self, faild_version_data):
+# get emerge info
+destfile = yield os.path.join(self.masterdest, 'emerge_info.txt')
+sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
+self.addFileUploade(sourcefile, destfile)
+#FIXME:
+# if faild_version_data:
+# get emerge.log
+# get elogs
+# world file
+
+def getBuildWorkdirFiles(self):
+#FIXME:
+# get files from the build workdir
+pass
+
 @defer.inlineCallbacks
 def run(self):
 self.gentooci = 
self.master.namedServices['services'].namedServices['gentooci']
@@ -610,6 +631,8 @@ class CheckEmergeLogs(BuildStep):
 ]
 package_dict = emerge_output['packages']
 
+self.masterdest = yield os.path.join(self.master.basedir, 'workers', 
self.getProperty('workername'), str(self.getProperty("buildnumber")))
+
 #FIXME: Prosees the logs and do stuff
 # preserved-libs
 if emerge_output['preserved_libs'] and 
projects_emerge_options['preserved_libs']:
@@ -757,12 +780,12 @@ class CheckEmergeLogs(BuildStep):
 print(log_dict)
 cpv = self.getProperty('cpv')
 faild_cpv = emerge_output['failed']
+faild_version_data = False
 if cpv in log_dict or faild_cpv in log_dict:
+yield self.createDistDir()
 if cpv in log_dict:
 self.log_data[cpv] = log_dict[cpv]
-yield self.createDistDir()
 yield self.getLogFile(cpv, log_dict)
-faild_version_data = False
 if faild_cpv:
 # failed and build requested cpv
 if cpv == faild_cpv:
@@ -772,6 +795,8 @@ class CheckEmergeLogs(BuildStep):
 self.log_data[faild_cpv] = log_dict[faild_cpv]
 yield self.getLogFile(faild_cpv, log_dict)
 faild_version_data = yield 
self.getVersionData(faild_cpv)
+self.getBuildWorkdirFiles()
+self.getEmergeFiles(faild_version_data)
 self.aftersteps_list.append(steps.Trigger(
 schedulerNames=['parse_build_log'],
 waitForFinish=False,
@@ -785,7 +810,6 @@ class CheckEmergeLogs(BuildStep):
 'reposit

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2021-10-07 Thread Magnus Granberg 
commit: 66b8e3dc0f0d56d3a154353b4363efa65850e4d1
Author: Magnus Granberg  gentoo  org>
AuthorDate: Thu Oct  7 09:18:11 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Thu Oct  7 09:18:11 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=66b8e3dc

Fix support for bb 3.3.0

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  2 +-
 buildbot_gentoo_ci/steps/builders.py   | 22 +++---
 buildbot_gentoo_ci/steps/portage.py|  2 +-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 41c3171..45f8f48 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -101,7 +101,7 @@ def run_build_request():
 f.addStep(builders.SetupPropertys())
 # Clean and add new /etc/portage
 #NOTE: remove the symlink befor the dir
-f.addStep(buildbot_steps.ShellCommandNewStyle(
+f.addStep(buildbot_steps.ShellCommand(
 command=['rm', 'make.profile'],
 workdir='/etc/portage/'
 ))

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 80fe52f..79be77f 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -375,7 +375,7 @@ class RunEmerge(BuildStep):
 shell_commad_list.append('--buildpkg-exclude')
 shell_commad_list.append('acct-*')
 aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
 command=shell_commad_list,
 strip=True,
 extract_fn=PersOutputOfEmerge,
@@ -394,7 +394,7 @@ class RunEmerge(BuildStep):
 shell_commad_list.append('--buildpkg-exclude')
 shell_commad_list.append('acct-*')
 aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
 command=shell_commad_list,
 strip=True,
 extract_fn=PersOutputOfEmerge,
@@ -409,7 +409,7 @@ class RunEmerge(BuildStep):
 shell_commad_list.append('-q')
 shell_commad_list.append('@preserved-rebuild')
 aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
 command=shell_commad_list,
 strip=True,
 extract_fn=PersOutputOfEmerge,
@@ -423,7 +423,7 @@ class RunEmerge(BuildStep):
 shell_commad_list.append('--pretend')
 shell_commad_list.append('--depclean')
 aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
 command=shell_commad_list,
 strip=True,
 extract_fn=PersOutputOfDepclean,
@@ -439,7 +439,7 @@ class RunEmerge(BuildStep):
 if self.getProperty('depclean'):
 pass
 aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
 command=shell_commad_list,
 strip=True,
 extract_fn=PersOutputOfDepclean,
@@ -468,7 +468,7 @@ class RunEmerge(BuildStep):
 shell_commad_list.append('--usepkg=n')
 shell_commad_list.append(c + '/' + p)
 aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
 command=shell_commad_list,
 strip=True,
 extract_fn=PersOutputOfEmerge,
@@ -492,7 +492,7 @@ class RunEmerge(BuildStep):
 shell_commad_list.append('acct-*')
 shell_commad_list.append('-p')
 aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
 command=shell_commad_list,
 strip=True,
 extract_fn=PersOutputOfEmerge,
@@ -517,7 +517,7 @@ class RunEmerge(BuildStep):
 shell_commad_list.append('--buildpkg-exclude')
 shell_commad_list.append('acct-*')
 aftersteps_list.append(
-steps.SetPropertyFromCommandNewStyle(
+steps.SetPropertyFromCommand(
 command=shell_commad_list,
 strip=True,
 extract_fn=PersOutputOfEmerge,
@@ -663,7 +663,7 @@ class CheckEmergeLogs(BuildStep):
 shell_commad_list.a

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2021-05-02 Thread Magnus Granberg 
commit: c5da0d10e98b62390d47b5b13d66a135706177ed
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun May  2 15:24:05 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun May  2 15:24:05 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c5da0d10

Add support for emerge --info

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  2 ++
 buildbot_gentoo_ci/steps/builders.py   | 44 +-
 buildbot_gentoo_ci/steps/logs.py   | 24 +++-
 3 files changed, 68 insertions(+), 2 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index e08127b..417e385 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -177,6 +177,8 @@ def parse_build_log():
 f.addStep(logs.MakeIssue())
 # add sum log to buildbot log
 f.addStep(logs.setBuildbotLog())
+# add emerge info to log and db
+f.addStep(logs.setEmergeInfoLog())
 # set BuildStatus
 f.addStep(logs.setBuildStatus())
 # setup things for the irc bot

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 50f924d..3a5de47 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -164,6 +164,17 @@ def PersOutputOfDepclean(rc, stdout, stderr):
 'depclean_output' : depclean_output
 }
 
+def PersOutputOfEmergeInfo(rc, stdout, stderr):
+emerge_info_output = {}
+emerge_info_output['rc'] = rc
+emerge_info_list = []
+for line in stdout.split('\n'):
+emerge_info_list.append(line)
+emerge_info_output['emerge_info'] = emerge_info_list
+return {
+'emerge_info_output' : emerge_info_output
+}
+
 class TriggerRunBuildRequest(BuildStep):
 
 name = 'TriggerRunBuildRequest'
@@ -650,7 +661,8 @@ class CheckEmergeLogs(BuildStep):
 'pkg_check_log_data' : 
self.getProperty("pkg_check_log_data"),
 'repository_data' : 
self.getProperty('repository_data'),
 'faild_cpv' : faild_cpv,
-'step' : self.step
+'step' : self.step,
+'emerge_info' : 
self.getProperty('emerge_info_output')['emerge_info']
 }
 ))
 if not self.step is None and self.aftersteps_list != []:
@@ -761,6 +773,35 @@ class CheckPkgCheckLogs(BuildStep):
 self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
 return SUCCESS
 
+class RunEmergeInfo(BuildStep):
+
+name = 'RunEmergeInfo'
+description = 'Running'
+descriptionDone = 'Ran'
+haltOnFailure = True
+flunkOnFailure = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+aftersteps_list = []
+shell_commad_list = [
+'emerge',
+]
+shell_commad_list.append('--info')
+aftersteps_list.append(
+steps.SetPropertyFromCommandNewStyle(
+command=shell_commad_list,
+strip=True,
+extract_fn=PersOutputOfEmergeInfo,
+workdir='/',
+timeout=None
+))
+yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+return SUCCESS
+
 class RunBuild(BuildStep):
 
 name = 'RunBuild'
@@ -780,6 +821,7 @@ class RunBuild(BuildStep):
 # trigger pars_build_log if we have any logs to check
 return SUCCESS
 aftersteps_list = []
+aftersteps_list.append(RunEmergeInfo())
 aftersteps_list.append(RunEmerge(step='pre-build'))
 aftersteps_list.append(RunEmerge(step='build'))
 self.setProperty('depclean', False, 'depclean')

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 0960b9b..3c0ef8d 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -252,7 +252,29 @@ class setBuildbotLog(BuildStep):
 # add line for line
 for line in self.getProperty('summary_log_list'):
 yield log.addStdout(line + '\n')
-# add emerge info log
+return SUCCESS
+
+class setEmergeInfoLog(BuildStep):
+
+name = 'setEmergeInfoLog'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = False
+flunkOnFailure = True
+warnOnWarnings = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+#setup the log
+log = yield self.addLog('emerge_info')
+#FIXME: add emerge info to db
+# add line for line
+for

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2021-04-04 Thread Magnus Granberg 
commit: 81e7f0bdd56bf11fdbc0990191273ad59c1a3b01
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun Apr  4 20:29:38 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun Apr  4 20:29:38 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=81e7f0bd

Add support for Change USE, one use only

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  2 +-
 buildbot_gentoo_ci/steps/builders.py   | 58 ++
 buildbot_gentoo_ci/steps/portage.py|  1 +
 3 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 6e8baf6..fddfa2b 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -120,7 +120,7 @@ def run_build_request():
 # setup env
 f.addStep(portage.SetEnvDefault())
 # setup package.*
-#f.addStep(portage.SetPackageDefault())
+f.addStep(portage.SetPackageDefault())
 # setup files in /etc if needed
 # run --regen if needed on repo
 # update packages before any tests

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index b28d7e5..96572c0 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -19,6 +19,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
 emerge_output['rc'] = rc
 emerge_output['preserved_libs'] = False
 emerge_output['depclean'] = False
+emerge_output['change_use'] = False
 package_dict = {}
 log_path_list = []
 print(stderr)
@@ -82,7 +83,20 @@ def PersOutputOfEmerge(rc, stdout, stderr):
 # split the lines
 #FIXME: Handling of stderr output
 for line in stderr.split('\n'):
-pass
+if 'Change USE:' in line:
+line_list = line.split(' ')
+change_use_list = []
+# get cpv
+cpv_split = line_list[1].split(':')
+change_use_list.append(cpv_split[0])
+# add use flags
+if line_list[4].startswith('+') or line_list[4].startswith('-'):
+# we only support one for now
+if line_list[4].endswith(')'):
+change_use_list.append(line_list[4].replace(')', ''))
+else:
+change_use_list = False
+emerge_output['change_use'] = change_use_list
 return {
 'emerge_output' : emerge_output
 }
@@ -209,6 +223,7 @@ class SetupPropertys(BuildStep):
 self.setProperty('cpv_build', False, 'cpv_build')
 self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
 self.setProperty('faild_version_data', None, 'faild_version_data')
+self.setProperty('rerun', 0, 'rerun')
 print(self.getProperty("buildnumber"))
 if self.getProperty('project_build_data') is None:
 project_build_data = {}
@@ -268,13 +283,13 @@ class RunEmerge(BuildStep):
 name = 'RunEmerge'
 description = 'Running'
 descriptionDone = 'Ran'
-descriptionSuffix = None
 haltOnFailure = True
 flunkOnFailure = True
 
 def __init__(self, step=None,**kwargs):
 self.step = step
 super().__init__(**kwargs)
+self.descriptionSuffix = self.step
 
 @defer.inlineCallbacks
 def run(self):
@@ -388,7 +403,6 @@ class RunEmerge(BuildStep):
 cpv = self.getProperty("cpv")
 c = yield catpkgsplit(cpv)[0]
 p = yield catpkgsplit(cpv)[1]
-shell_commad_list.append('-p')
 shell_commad_list.append('=' + self.getProperty('cpv'))
 # we don't use the bin for the requsted cpv
 shell_commad_list.append('--usepkg-exclude')
@@ -398,6 +412,7 @@ class RunEmerge(BuildStep):
 shell_commad_list.append('virtual')
 shell_commad_list.append('--buildpkg-exclude')
 shell_commad_list.append('acct-*')
+shell_commad_list.append('-p')
 aftersteps_list.append(
 steps.SetPropertyFromCommandNewStyle(
 command=shell_commad_list,
@@ -444,13 +459,13 @@ class CheckEmergeLogs(BuildStep):
 name = 'CheckLogs'
 description = 'Running'
 descriptionDone = 'Ran'
-descriptionSuffix = None
 haltOnFailure = True
 flunkOnFailure = True
 
 def __init__(self, step=None,**kwargs):
 self.step = step
 super().__init__(**kwargs)
+self.descriptionSuffix = self.step
 
 @defer.inlineCallbacks
 def run(self):
@@ -480,10 +495,41 @@ class CheckEmergeLogs(BuildStep):
 print(self.getProperty('cpv_build'))
 
 #FIXME:
-# update package.* if needed and rerun pre-build max X times
+# update package.* if needed and rerun pre-build max 3 times
 if self.step == 'pre-build':
 print(emerge_output)
-
+

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2021-04-04 Thread Magnus Granberg 
commit: 915abba59f59767d341bb190de5bba5ae03097d0
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun Apr  4 20:26:01 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun Apr  4 20:26:01 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=915abba5

Add build log sammary

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  2 +-
 buildbot_gentoo_ci/steps/logs.py   | 35 +-
 2 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index d8db8f8..6e8baf6 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -177,7 +177,7 @@ def parse_build_log():
 # set it SUCCESS/FAILURE/WARNINGS
 f.addStep(logs.MakeIssue())
 # add sum log to buildbot log
-#f.addStep(logs.setBuildbotLog)
+f.addStep(logs.setBuildbotLog())
 # set BuildStatus
 f.addStep(logs.setBuildStatus())
 # setup things for the irc bot

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 5fe248f..767fdd4 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -175,7 +175,7 @@ class ParserBuildLog(BuildStep):
 print(self.summery_dict)
 # remove all lines with ignore in the dict
 # setProperty summery_dict
-self.setProperty("summery_log_dict", self.summery_dict, 
'summery_log_dict')
+self.setProperty("summary_log_dict", self.summery_dict, 
'summary_log_dict')
 return SUCCESS
 
 class MakeIssue(BuildStep):
@@ -193,27 +193,50 @@ class MakeIssue(BuildStep):
 #@defer.inlineCallbacks
 def run(self):
 self.gentooci = 
self.master.namedServices['services'].namedServices['gentooci']
-summery_log_dict = self.getProperty('summery_log_dict')
+summary_log_dict = self.getProperty('summary_log_dict')
 error = False
 warning = False
-self.summery_log_list = []
+self.summary_log_list = []
 log_hash = hashlib.sha256()
-for k, v in sorted(summery_log_dict.items()):
+for k, v in sorted(summary_log_dict.items()):
 if v['status'] == 'error':
 error = True
 if v['status'] == 'warning':
 warning = True
-self.summery_log_list.append(v['text'])
+self.summary_log_list.append(v['text'])
 log_hash.update(v['text'].encode('utf-8'))
 # add build log
 # add issue/bug/pr report
-self.setProperty("summery_log_list", self.summery_log_list, 
'summery_log_list')
+self.setProperty("summary_log_list", self.summary_log_list, 
'summary_log_list')
 if error:
 self.setProperty("status", 'failed', 'status')
 if warning:
 self.setProperty("status", 'warning', 'status')
 return SUCCESS
 
+class setBuildbotLog(BuildStep):
+
+name = 'setBuildbotLog'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = False
+flunkOnFailure = True
+warnOnWarnings = True
+
+def __init__(self, **kwargs):
+super().__init__(**kwargs)
+
+@defer.inlineCallbacks
+def run(self):
+#setup the log
+log = yield self.addLog('summary')
+# add line for line
+for line in self.getProperty('summary_log_list'):
+yield log.addStdout(line + '\n')
+# add emerge info log
+return SUCCESS
+
 class setBuildStatus(BuildStep):
 
 name = 'setBuildStatus'

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2021-03-24 Thread Magnus Granberg 
commit: b720c6bbcb79211a276d06a1eaf1543b89644767
Author: Magnus Granberg  gentoo  org>
AuthorDate: Wed Mar 24 23:14:47 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Wed Mar 24 23:14:47 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b720c6bb

Add the log parser BuildFactory steps

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py  |  8 +++
 buildbot_gentoo_ci/config/buildfactorys.py | 22 +++-
 buildbot_gentoo_ci/config/schedulers.py|  3 +
 buildbot_gentoo_ci/steps/builders.py   | 90 --
 buildbot_gentoo_ci/steps/portage.py| 10 +++-
 5 files changed, 101 insertions(+), 32 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index c7b4469..0235753 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -49,4 +49,12 @@ def gentoo_builders(b=[]):
 factory=buildfactorys.run_build_request()
 )
 )
+# FIXME: get workers from db
+# Use multiplay workers
+b.append(util.BuilderConfig(
+name='parse_build_log',
+workername='updatedb_1',
+factory=buildfactorys.parse_build_log()
+)
+)
 return b

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index db48130..df7baef 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -10,6 +10,7 @@ from buildbot_gentoo_ci.steps import package
 from buildbot_gentoo_ci.steps import version
 from buildbot_gentoo_ci.steps import builders
 from buildbot_gentoo_ci.steps import portage
+from buildbot_gentoo_ci.steps import logs
 
 def update_db_check():
 f = util.BuildFactory()
@@ -159,5 +160,24 @@ def run_build_request():
 f.addStep(builders.RunEmerge(step='depclean'))
 f.addStep(builders.RunEmerge(step='preserved-libs'))
 f.addStep(builders.RunEmerge(step='depclean'))
-f.addStep(builders.setBuildStatus())
+return f
+
+def parse_build_log():
+f = util.BuildFactory()
+# FIXME: 6
+# set needed Propertys
+f.addStep(logs.SetupPropertys())
+# pers the build log for info qa errors
+#f.addStep(logs.ParserBuildLog())
+# pers the log from pkg check
+#f.addStep(logs.ParserPkgCheckLog())
+# Upload the log to the cloud and remove the log
+#f.addStep(logs.Upload())
+# check the sum log if we need to make a issue/bug/pr report
+# set it SUCCESS/FAILURE/WARNINGS
+#f.addStep(logs.MakeIssue())
+# set BuildStatus
+#f.addStep(logs.setBuildStatus())
+# setup things for the irc bot
+#f.addStep(logs.SetIrcInfo())
 return f

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index 0cbec96..efdac75 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -74,6 +74,8 @@ def gentoo_schedulers():
builderNames=["build_request_data"])
 run_build_request = schedulers.Triggerable(name="run_build_request",
builderNames=["run_build_request"])
+pers_build_log = schedulers.Triggerable(name="parse_build_log",
+   builderNames=["parse_build_log"])
 s = []
 s.append(test_updatedb)
 s.append(scheduler_update_db)
@@ -81,4 +83,5 @@ def gentoo_schedulers():
 s.append(update_v_data)
 s.append(build_request_data)
 s.append(run_build_request)
+s.append(parse_build_log)
 return s

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index 22608e3..ff6dd94 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -20,22 +20,26 @@ def PersOutputOfEmerge(rc, stdout, stderr):
 emerge_output['preserved_libs'] = False
 emerge_output['depclean'] = False
 package_dict = {}
+log_path_list = []
 print(stderr)
 emerge_output['stderr'] = stderr
 # split the lines
 for line in stdout.split('\n'):
 # package list
 subdict = {}
-if line.startswith('[ebuild') or line.startswith('[binary'):
+subdict2 = {}
+if line.startswith('[ebuild') or line.startswith('[binary') or 
line.startswith('[nomerge'):
 # if binaries
-if line.startswith('[ebuild'):
+if line.startswith('[ebuild') or line.startswith('[nomerge'):
 subdict['binary'] = False
 else:
 subdict['binary'] = True
 # action [ N ] stuff
 subdict['action'] = line[8:15].replace(' ', '')
 # cpv
+#FIXME: We my have more then one spece betvine ] and cpv
 cpv_split = re.search('] (.+?) ', line).group(1).split(':')
+print(cpv_split)
 cpv = cpv_split[0]
 # repositor

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

2021-03-18 Thread Magnus Granberg 
commit: 04836c8c78369e1a5acbc460343b003c3a337f20
Author: Magnus Granberg  gentoo  org>
AuthorDate: Thu Mar 18 23:16:08 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Thu Mar 18 23:16:08 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=04836c8c

Clean up Package and Category code

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  8 +++---
 buildbot_gentoo_ci/steps/category.py   | 25 ++---
 buildbot_gentoo_ci/steps/package.py| 44 +-
 3 files changed, 44 insertions(+), 33 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index f5a3b35..db48130 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -41,17 +41,17 @@ def update_db_cp():
 # else
 #   add category to db step
 #   return category_data
-f.addStep(category.CheckCGentooCiProject())
+f.addStep(category.CheckC())
 # if package in db
 #   return package_data
 #   add check package path step at end
 # else
 #   add package to db step
 #   return package_data
-f.addStep(package.CheckPGentooCiProject())
+f.addStep(package.CheckP())
 # Trigger new builders with v from cpv
-#   return package_data, cpv, repository_data, project_data, config_root
-f.addStep(package.TriggerCheckVGentooCiProject())
+#   return package_data, cpv, repository_data, project_data
+f.addStep(package.TriggerCheckForV())
 return f
 
 def update_db_v():

diff --git a/buildbot_gentoo_ci/steps/category.py 
b/buildbot_gentoo_ci/steps/category.py
index 1aca99a..81061e6 100644
--- a/buildbot_gentoo_ci/steps/category.py
+++ b/buildbot_gentoo_ci/steps/category.py
@@ -18,6 +18,13 @@ class AddCategory(BuildStep):
 def __init__(self, **kwargs):
 super().__init__(**kwargs)
 
+name = 'AddCategory'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
 @defer.inlineCallbacks
 def run(self):
 self.gentooci = 
self.master.namedServices['services'].namedServices['gentooci']
@@ -26,27 +33,25 @@ class AddCategory(BuildStep):
 self.category_data['uuid'] = yield 
self.gentooci.db.categorys.addCategory(self.category_data['name'])
 print(self.category_data)
 self.setProperty("category_data", self.category_data, 'category_data')
-self.setProperty("config_root", self.getProperty("config_root"), 
'config_root')
-self.setProperty("project_data", self.getProperty("project_data"), 
'project_data')
-self.setProperty("repository_data", 
self.getProperty("repository_data"), 'repository_data')
-self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
 return SUCCESS
 
-class CheckCGentooCiProject(BuildStep):
+class CheckC(BuildStep):
 def __init__(self, **kwargs):
 super().__init__(**kwargs)
 
+name = 'CheckC'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
 @defer.inlineCallbacks
 def run(self):
 self.gentooci = 
self.master.namedServices['services'].namedServices['gentooci']
 self.category = yield self.getProperty("cpv").split('/')[0]
 print(self.category)
-print(self.getProperty("repository"))
 self.category_data = yield 
self.gentooci.db.categorys.getCategoryByName(self.category)
-self.setProperty("config_root", self.getProperty("config_root"), 
'config_root')
-self.setProperty("project_data", self.getProperty("project_data"), 
'project_data')
-self.setProperty("repository_data", 
self.getProperty("repository_data"), 'repository_data')
-self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
 print(self.category_data)
 if self.category_data is None:
 self.setProperty("category", self.category, 'category')

diff --git a/buildbot_gentoo_ci/steps/package.py 
b/buildbot_gentoo_ci/steps/package.py
index 3a28da2..ea5ab23 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -20,6 +20,13 @@ class AddPackage(BuildStep):
 def __init__(self, **kwargs):
 super().__init__(**kwargs)
 
+name = 'AddPackage'
+description = 'Running'
+descriptionDone = 'Ran'
+descriptionSuffix = None
+haltOnFailure = True
+flunkOnFailure = True
+
 @defer.inlineCallbacks
 def run(self):
 self.gentooci = 
self.master.namedServices['services'].namedServices['gentooci']
@@ -34,28 +41,25 @@ class AddPackage(BuildStep):
 )
 print(self.package_data)
 self.setProperty("package_data", self.package_data, 'package_data')
-self.setProperty("config_root", self.g

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/

2021-02-22 Thread Magnus Granberg 
commit: 4324b82edd9d69a27b1db9f4272564ddc51a75dd
Author: Magnus Granberg  gentoo  org>
AuthorDate: Mon Feb 22 20:27:14 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Mon Feb 22 20:27:14 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4324b82e

Add RunPkgCheck and CheckPkgCheckLogs

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  11 ++-
 buildbot_gentoo_ci/db/model.py |   2 +-
 buildbot_gentoo_ci/db/projects.py  |   6 +-
 buildbot_gentoo_ci/steps/builders.py   | 120 +++--
 4 files changed, 129 insertions(+), 10 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 3c77762..cfe4131 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -92,13 +92,15 @@ def run_build_request():
 # set needed Propertys
 f.addStep(builders.SetupPropertys())
 # Clean and add new /etc/portage
+#FIXME: Is don't like symlinks
 f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
 workdir='/etc/'))
 f.addStep(buildbot_steps.MakeDirectory(dir="portage",
 workdir='/etc/'))
 # setup the profile
-f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
-workdir='/etc/portage/'))
+#NOTE: pkgcheck do not support it as a dir
+#f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
+#workdir='/etc/portage/'))
 f.addStep(builders.SetMakeProfile())
 # setup repos.conf dir
 f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
@@ -132,4 +134,9 @@ def run_build_request():
 #   check log
 # setup make.conf if build id has changes make.conf as dict from 
SetMakeConf
 # setup package.* env if build id has changes
+# setup pkgcheck.conf if needed
+#f.addStep(builders.SetPkgCheckConf())
+# run pkgcheck if wanted
+#   check log
+f.addStep(builders.RunPkgCheck())
 return f

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 0a2c145..4d691ea 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -128,7 +128,7 @@ class Model(base.DBConnectorComponent):
   sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
   nullable=False),
 sa.Column('auto', sa.Boolean, default=False),
-sa.Column('pkgcheck', sa.Boolean, default=False),
+sa.Column('pkgcheck', sa.Enum('package','full','none'), 
default='none'),
 )
 
 # projects etc/portage settings

diff --git a/buildbot_gentoo_ci/db/projects.py 
b/buildbot_gentoo_ci/db/projects.py
index eb8f727..9aeca50 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -160,12 +160,16 @@ class 
ProjectsConnectorComponent(base.DBConnectorComponent):
 )
 
 def _row2dict_projects_repositorys(self, conn, row):
+if row.pkgcheck == 'none':
+pkgcheck = False
+else:
+pkgcheck=row.pkgcheck
 return dict(
 id=row.id,
 project_uuid=row.project_uuid,
 repository_uuid=row.repository_uuid,
 auto=row.auto,
-pkgcheck=row.pkgcheck
+pkgcheck=pkgcheck
 )
 
 def _row2dict_projects_portage(self, conn, row):

diff --git a/buildbot_gentoo_ci/steps/builders.py 
b/buildbot_gentoo_ci/steps/builders.py
index c0567c1..6d46ae6 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,6 +4,8 @@
 import os
 import re
 
+from portage.versions import catpkgsplit
+
 from twisted.internet import defer
 from twisted.python import log
 
@@ -75,6 +77,26 @@ def PersOutputOfEmerge(rc, stdout, stderr):
 'emerge_output' : emerge_output
 }
 
+def PersOutputOfPkgCheck(rc, stdout, stderr):
+pkgcheck_output = {}
+pkgcheck_output['rc'] = rc
+#FIXME: Handling of stdout output
+pkgcheck_xml_list = []
+# split the lines
+for line in stdout.split('\n'):
+#  pkgcheck output list
+if line.startswith('

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/

2021-01-17 Thread Magnus Granberg 
commit: 81da08ad4476408915e0e082a33f5a1c2ebe159e
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun Jan 17 09:25:07 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun Jan 17 09:25:07 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=81da08ad

Add BuildFactory update_db_v

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/builders.py  |  16 ++
 buildbot_gentoo_ci/config/buildfactorys.py |  36 +++-
 buildbot_gentoo_ci/config/schedulers.py|   6 +
 buildbot_gentoo_ci/db/connector.py |   4 +
 buildbot_gentoo_ci/db/keywords.py  |  76 +
 buildbot_gentoo_ci/db/model.py |  28 +--
 buildbot_gentoo_ci/db/versions.py  | 112 
 buildbot_gentoo_ci/steps/package.py|  34 +++-
 buildbot_gentoo_ci/steps/update_db.py  |   6 +-
 buildbot_gentoo_ci/steps/version.py| 264 +
 10 files changed, 562 insertions(+), 20 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py 
b/buildbot_gentoo_ci/config/builders.py
index c069841..9fef46a 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -22,4 +22,20 @@ def gentoo_builders(b=[]):
 factory=buildfactorys.update_db_cp()
 )
 )
+# FIXME: get workers from db
+# Use multiplay workers
+b.append(util.BuilderConfig(
+name='update_v_data',
+workername='updatedb_1',
+factory=buildfactorys.update_db_v()
+)
+)
+# FIXME: get workers from db
+# Use multiplay workers
+b.append(util.BuilderConfig(
+name='build_request_data',
+workername='updatedb_1',
+factory=buildfactorys.build_request_check()
+)
+)
 return b

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 4038bb1..ba96e5a 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -7,6 +7,7 @@ from buildbot.plugins import util
 from buildbot_gentoo_ci.steps import update_db
 from buildbot_gentoo_ci.steps import category
 from buildbot_gentoo_ci.steps import package
+from buildbot_gentoo_ci.steps import version
 
 def update_db_check():
 f = util.BuildFactory()
@@ -45,7 +46,36 @@ def update_db_cp():
 #   add package to db step
 #   return package_data
 f.addStep(package.CheckPGentooCiProject())
-# Make a for loop and trigger new builders with v from cpv
-#   return package_data, cpv, repository, project_data, config_root
-#f.addStep(package.TriggerVGentooCiProject())
+# Trigger new builders with v from cpv
+#   return package_data, cpv, repository_data, project_data, config_root
+f.addStep(package.TriggerCheckVGentooCiProject())
+return f
+
+def update_db_v():
+f = util.BuildFactory()
+# FIXME: 3
+# if version in db
+#   return version_data
+f.addStep(version.GetVData())
+#   check path and hash
+f.addStep(version.CheckPathHash())
+#   if not path
+#   if not hash
+#   add deleted stage att end
+#   add version to db stage
+#   add version metadata to db
+#   add version to build check
+#   else
+#   add deleted stage att end
+#   add version to build check stage att end
+# else
+#   add version to db
+#   add version metadata to db
+#   add version to build check
+f.addStep(version.CheckV())
+return f
+
+def build_request_check():
+f = util.BuildFactory()
+# FIXME: 4
 return f

diff --git a/buildbot_gentoo_ci/config/schedulers.py 
b/buildbot_gentoo_ci/config/schedulers.py
index 88d0e49..687b52f 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -52,8 +52,14 @@ def gentoo_schedulers():
 ])
 update_cpv_data = schedulers.Triggerable(name="update_cpv_data",
builderNames=["update_cpv_data"])
+update_v_data = schedulers.Triggerable(name="update_v_data",
+   builderNames=["update_v_data"])
+build_request_data = schedulers.Triggerable(name="build_request_data",
+   builderNames=["build_request_data"])
 s = []
 s.append(test_updatedb)
 #s.append(scheduler_update_db)
 s.append(update_cpv_data)
+s.append(update_v_data)
+s.append(build_request_data)
 return s

diff --git a/buildbot_gentoo_ci/db/connector.py 
b/buildbot_gentoo_ci/db/connector.py
index 0121d11..35febe6 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -34,6 +34,8 @@ from buildbot_gentoo_ci.db import projects
 from buildbot_gentoo_ci.db import repositorys
 from buildbot_gentoo_ci.db import categorys
 from buildbot_gentoo_ci.db import packages
+from buildbot_gentoo_ci.db import versions
+from buildbot_gentoo_ci.db impo

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/

2021-01-11 Thread Magnus Granberg 
commit: a4a5adaa76b48ae04c142f0c1bf602c210892731
Author: Magnus Granberg  gentoo  org>
AuthorDate: Mon Jan 11 17:45:46 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Mon Jan 11 17:45:46 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=a4a5adaa

Add CheckPGentooCiProject on BuildFactory update_db_cp

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  5 +-
 buildbot_gentoo_ci/db/connector.py |  2 +
 buildbot_gentoo_ci/db/packages.py  | 80 ++
 buildbot_gentoo_ci/steps/category.py   | 10 ++--
 buildbot_gentoo_ci/steps/package.py| 65 
 buildbot_gentoo_ci/steps/update_db.py  | 30 +--
 6 files changed, 170 insertions(+), 22 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 4a0ffcd..4038bb1 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -6,7 +6,7 @@ from buildbot.plugins import util
 
 from buildbot_gentoo_ci.steps import update_db
 from buildbot_gentoo_ci.steps import category
-#from buildbot_gentoo_ci.steps import package
+from buildbot_gentoo_ci.steps import package
 
 def update_db_check():
 f = util.BuildFactory()
@@ -44,7 +44,8 @@ def update_db_cp():
 # else
 #   add package to db step
 #   return package_data
-#f.addStep(package.CheckPGentooCiProject())
+f.addStep(package.CheckPGentooCiProject())
 # Make a for loop and trigger new builders with v from cpv
 #   return package_data, cpv, repository, project_data, config_root
+#f.addStep(package.TriggerVGentooCiProject())
 return f

diff --git a/buildbot_gentoo_ci/db/connector.py 
b/buildbot_gentoo_ci/db/connector.py
index c10fc49..0121d11 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -33,6 +33,7 @@ from buildbot_gentoo_ci.db import model
 from buildbot_gentoo_ci.db import projects
 from buildbot_gentoo_ci.db import repositorys
 from buildbot_gentoo_ci.db import categorys
+from buildbot_gentoo_ci.db import packages
 
 upgrade_message = textwrap.dedent("""\
 
@@ -75,6 +76,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
 self.projects = projects.ProjectsConnectorComponent(self)
 self.repositorys = repositorys.RepositorysConnectorComponent(self)
 self.categorys = categorys.CategorysConnectorComponent(self)
+self.packages = packages.PackagesConnectorComponent(self)
 
 @defer.inlineCallbacks
 def setup(self, config, check_version=True, verbose=True):

diff --git a/buildbot_gentoo_ci/db/packages.py 
b/buildbot_gentoo_ci/db/packages.py
new file mode 100644
index 000..13cc267
--- /dev/null
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -0,0 +1,80 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors. 
+# Buildbot is free software: you can redistribute it and/or modify it 
+# under the terms of the GNU General Public License as published by the 
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class PackagesConnectorComponent(base.DBConnectorComponent):
+
+@defer.inlineCallbacks
+def getPackageByName(self, name):
+def thd(conn):
+tbl = self.db.model.packages
+q = tbl.select()
+q = q.where(tbl.c.name == name)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
+@defer.inlineCallbacks
+def getPackageByUuid(self, uuid):
+def thd(conn):
+tbl = self.db.model.packages
+q = tbl.select()
+q = q.where(tbl.c.uuid == uuid)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
+@defer.inlineCallbacks
+def addPackage(self, name, repository_uuid, category_uuid):
+def thd(conn, no_recurse=False):
+try:
+tbl = self.db.model.

[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/

2021-01-10 Thread Magnus Granberg 
commit: 12c4aa125889ef4518a7ecc8232a10d55b129301
Author: Magnus Granberg  gentoo  org>
AuthorDate: Sun Jan 10 21:40:48 2021 +
Commit: Magnus Granberg  gentoo  org>
CommitDate: Sun Jan 10 21:40:48 2021 +
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=12c4aa12

Add CheckCGentooCiProject on BuildFactory update_db_cp

Signed-off-by: Magnus Granberg  gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  4 +-
 buildbot_gentoo_ci/db/categorys.py | 76 ++
 buildbot_gentoo_ci/db/connector.py |  2 +
 buildbot_gentoo_ci/db/model.py | 10 ++--
 buildbot_gentoo_ci/steps/category.py   | 58 +++
 5 files changed, 143 insertions(+), 7 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py 
b/buildbot_gentoo_ci/config/buildfactorys.py
index 3fae376..4a0ffcd 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -5,7 +5,7 @@ from buildbot.plugins import steps as buildbot_steps
 from buildbot.plugins import util
 
 from buildbot_gentoo_ci.steps import update_db
-#from buildbot_gentoo_ci.steps import category
+from buildbot_gentoo_ci.steps import category
 #from buildbot_gentoo_ci.steps import package
 
 def update_db_check():
@@ -37,7 +37,7 @@ def update_db_cp():
 # else
 #   add category to db step
 #   return category_data
-#f.addStep(category.CheckCGentooCiProject())
+f.addStep(category.CheckCGentooCiProject())
 # if package in db
 #   return package_data
 #   add check package path step at end

diff --git a/buildbot_gentoo_ci/db/categorys.py 
b/buildbot_gentoo_ci/db/categorys.py
new file mode 100644
index 000..f2bfde4
--- /dev/null
+++ b/buildbot_gentoo_ci/db/categorys.py
@@ -0,0 +1,76 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors. 
+# Buildbot is free software: you can redistribute it and/or modify it 
+# under the terms of the GNU General Public License as published by the 
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class CategorysConnectorComponent(base.DBConnectorComponent):
+
+@defer.inlineCallbacks
+def getCategoryByName(self, name):
+def thd(conn):
+tbl = self.db.model.categorys
+q = tbl.select()
+q = q.where(tbl.c.name == name)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
+@defer.inlineCallbacks
+def getCategoryByUuid(self, uuid):
+def thd(conn):
+tbl = self.db.model.categorys
+q = tbl.select()
+q = q.where(tbl.c.uuid == uuid)
+res = conn.execute(q)
+row = res.fetchone()
+if not row:
+return None
+return self._row2dict(conn, row)
+res = yield self.db.pool.do(thd)
+return res
+
+@defer.inlineCallbacks
+def addCategory(self, name):
+def thd(conn, no_recurse=False):
+try:
+tbl = self.db.model.categorys
+q = tbl.insert()
+r = conn.execute(q, dict(name=name))
+except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+uuid = None
+else:
+uuid = r.inserted_primary_key[0]
+return uuid
+res = yield self.db.pool.do(thd)
+return res
+
+def _row2dict(self, conn, row):
+return dict(
+uuid=row.uuid,
+name=row.name
+)

diff --git a/buildbot_gentoo_ci/db/connector.py 
b/buildbot_gentoo_ci/db/connector.py
index f95a121..c10fc49 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -32,6 +32,7 @@ from buildbot.util import service
 from buildbot_gentoo_ci.db import model
 from buildbot_gentoo_ci.db import projects
 from buildbot_gentoo_ci.db import repositorys
+from buildbot_gentoo_ci.db import categorys
 
 upgrade_message = textwrap.dedent("""\
 
@@ -73,6 +74,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
 self.model = model.Model(self)
 self.proje