commit: f95dce7cfc33e1d5a0e606c5562cac59c2dc0fdd
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jun 26 21:52:10 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jun 26 21:52:10 2023 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f95dce7c
Add run_clean_db_request to clean db
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/builders.py | 29 +++++++++
buildbot_gentoo_ci/config/buildfactorys.py | 9 +++
buildbot_gentoo_ci/config/schedulers.py | 3 +
buildbot_gentoo_ci/steps/clean.py | 97 ++++++++++++++++++++++++++++++
buildbot_gentoo_ci/steps/package.py | 45 ++++++++++++++
5 files changed, 183 insertions(+)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index b3ddb16..584dfdf 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -20,6 +20,24 @@ def getWorkers(worker_type, workers):
worker_list.append(worker['uuid'])
return worker_list
+# check if we have same package_data
+@defer.inlineCallbacks
+def checkPackageData(master, builder, req1, req2):
+ canBeCollapsed = yield buildrequest.BuildRequest.canBeCollapsed(master,
req1, req2)
+ selfBuildset , otherBuildset = yield defer.gatherResults([
+ master.data.get(('buildsets', req1['buildsetid'])),
+ master.data.get(('buildsets', req2['buildsetid']))
+ ])
+ print(f"TestCollapsed")
+ print(f"canBeCollapsed: {canBeCollapsed}")
+ print(f"selfBuildset: {selfBuildset}")
+ print(f"otherBuildset: {otherBuildset}")
+ if canBeCollapsed and selfBuildset['parent_buildid'] != None and \
+ otherBuildset['parent_buildid'] != None:
+ return True
+ else:
+ return False
+
@defer.inlineCallbacks
def CanWorkerBuildProject(builder, wfb, request):
gentooci =
builder.master.namedServices['services'].namedServices['gentooci']
@@ -118,4 +136,15 @@ def gentoo_builders(worker_data):
factory=buildfactorys.run_build_images_request()
)
)
+ # Use multiplay workers
+ b.append(util.BuilderConfig(
+ name='run_clean_db_request',
+ workernames=getWorkersLocal('local', worker_data['local']),
+ # look builder so we only do one time
+ # look=
+ # if we allready runnin with same package_data properties then
skip/collapse
+ #collapseRequests=checkPackageData,
+ factory=buildfactorys.run_clean_db_request()
+ )
+ )
return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index b47a209..6755308 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -14,6 +14,7 @@ from buildbot_gentoo_ci.steps import portage
from buildbot_gentoo_ci.steps import logs
from buildbot_gentoo_ci.steps import repos
from buildbot_gentoo_ci.steps import nodes
+from buildbot_gentoo_ci.steps import clean
def update_db_check():
f = util.BuildFactory()
@@ -66,6 +67,8 @@ def update_db_cpv():
f.addStep(package.TriggerCheckForV())
# update metadata if needed
f.addStep(package.CheckMetadataPackagePath())
+ # clean package db if needed (clean up ebuilds in db)
+ f.addStep(package.TriggerCleanPackageDb())
return f
def update_db_v():
@@ -201,3 +204,9 @@ def run_build_images_request():
# set the needed steps for making the image
f.addStep(nodes.SetupBuildTypeAndSteps())
return f
+
+def run_clean_db_request():
+ f = util.BuildFactory()
+ # set needed Propertys
+ f.addStep(clean.SetupPropertys())
+ return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index 7833233..c34d714 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -92,6 +92,8 @@ def gentoo_schedulers():
builderNames=["parse_build_log"])
run_build_images_request =
schedulers.Triggerable(name="run_build_images_request",
builderNames=["run_build_images_request"])
+ run_clean_db_request = schedulers.Triggerable(name="run_clean_db_request",
+ builderNames=["run_clean_db_request"])
s = []
s.append(create_images)
s.append(scheduler_update_db)
@@ -101,4 +103,5 @@ def gentoo_schedulers():
s.append(build_request_data)
s.append(run_build_request)
s.append(parse_build_log)
+ s.append(run_clean_db_request)
return s
diff --git a/buildbot_gentoo_ci/steps/clean.py
b/buildbot_gentoo_ci/steps/clean.py
new file mode 100644
index 0000000..73733f1
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/clean.py
@@ -0,0 +1,97 @@
+# Copyright 2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
+from buildbot.plugins import steps, util
+
+class SetupPropertys(BuildStep):
+ name = 'Setup propertys for clean db'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ aftersteps_list = []
+ # Check on deleted ebuild on package
+ deleted_ebuilds_data = yield
self.gentooci.db.versions.getEbuildsByPackage(self.getProperty("package_data")['uuid'],
deleted=True)
+ # check if older then X days
+ #FIXME: set days in config and move the check to db side
+ days = 30
+ days_in_s = 86400 * days
+ now = int(self.master.reactor.seconds())
+ for ebuild_data in deleted_ebuilds_data:
+ no_bug = True
+ # check if older the days
+ if ebuild_data['deleted_at'] < (now - days_in_s):
+ print(f"Ebuild : {ebuild_data['uuid']} will be removed")
+ # get build_data for deleted_ebuild_data['uuid']
+ build_data_list = yield
self.gentooci.db.builds.getBuildsByVersionUuid(ebuild_data['uuid'])
+ print(build_data_list)
+ for build_data in build_data_list:
+ if build_data['bug_id'] == 0:
+ # get buildsets
+ print(f"Build : {build_data['id']} will be removed")
+ yield
self.build.addStepsAfterCurrentStep([RemoveBuildFromDb(build_data)])
+ else:
+ no_bug = False
+ if no_bug:
+ yield self.build.
addStepsAfterLastStep([RemoveEbuildFromDb(ebuild_data)])
+ return SUCCESS
+
+class RemoveEbuildFromDb(BuildStep):
+ name = 'Clean ebuild from db'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, ebuild_data, **kwargs):
+ self.ebuild_data = ebuild_data
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ yield
self.gentooci.db.versions.removeVersionKeyword(self.ebuild_data['uuid'])
+ yield
self.gentooci.db.versions.removeVersionMetadata(self.ebuild_data['uuid'])
+ yield self.gentooci.db.versions.removeVersion(self.ebuild_data['uuid'])
+ self.descriptionDone = f"Ebuild : {self.ebuild_data['uuid']} will be
removed"
+ return SUCCESS
+
+class RemoveBuildFromDb(BuildStep):
+ name = 'Clean builds from db'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, build_data, **kwargs):
+ self.build_data = build_data
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ if self.build_data['buildbot_build_id'] != 0:
+ self.descriptionDone = f"BuildBot Build :
{self.build_data['buildbot_build_id']} will be removed"
+ bb_build_data = yield
self.master.db.builds.getBuild(self.build_data['buildbot_build_id'])
+ # remove buildbot data on db
+ # remove steps and logs
+ yield
self.master.db.logs.deleteLogChunks(self.build_data['buildbot_build_id'])
+ # remove propertys
+ # get buildsets
+ # remove build_data
+ yield self.gentooci.db.builds.removeBuild(self.build_data['id'])
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/package.py
b/buildbot_gentoo_ci/steps/package.py
index 13eafa2..847ba12 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -12,6 +12,7 @@ from twisted.python import log
from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
from buildbot.process import remotecommand
from buildbot.plugins import steps
@@ -282,3 +283,47 @@ class TriggerCheckForV(BuildStep):
)
yield self.build.addStepsAfterCurrentStep(addStepUpdateVData)
return SUCCESS
+
+class TriggerCleanPackageDb(BuildStep):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ name = 'TriggerCleanPackageDb'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ aftersteps_list = []
+ # Check on deleted ebuild on package
+ deleted_ebuilds_data = yield
self.gentooci.db.versions.getEbuildsByPackage(self.getProperty("package_data")['uuid'],
deleted=True)
+ # check if older then X days
+ #FIXME: set days in config and move the check to db side
+ days = 30
+ days_in_s = 86400 * days
+ needcleaning = 0
+ now = int(self.master.reactor.seconds())
+ for deleted_ebuild_data in deleted_ebuilds_data:
+ # check if older the days
+ if deleted_ebuild_data['deleted_at'] < (now - days_in_s):
+ needcleaning = needcleaning + 1
+ print(f"{str(needcleaning)} ebuilds need cleaning in db")
+ if needcleaning > 0:
+ aftersteps_list.append(
+ steps.Trigger(
+ schedulerNames=['run_clean_db_request'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'package_data' : self.getProperty("package_data"),
+ }
+ )
+ )
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ self.descriptionDone = f"Need DB Cleaning: Yes"
+ return SUCCESS
+ self.descriptionDone = f"Need DB Cleaning: No"
+ return SKIPPED
