commit: ee9e8401620340982d6e922ca982f50fee1ec6fa Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Sat Mar 20 13:34:03 2021 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Sat Mar 20 13:34:03 2021 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ee9e8401
Change GetCommitdata to use revision_data. Use SetEnvForEbuildSH to get auxdb data. Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org> buildbot_gentoo_ci/steps/master.py | 235 ++++++++++++++++++++++++++++++++++++ buildbot_gentoo_ci/steps/portage.py | 130 +++++++++++++++++++- buildbot_gentoo_ci/steps/version.py | 66 +++------- 3 files changed, 380 insertions(+), 51 deletions(-) diff --git a/buildbot_gentoo_ci/steps/master.py b/buildbot_gentoo_ci/steps/master.py new file mode 100644 index 0000000..dac3830 --- /dev/null +++ b/buildbot_gentoo_ci/steps/master.py @@ -0,0 +1,235 @@ +# This file has parts from Buildbot and is modifyed by Gentoo Authors. +# Buildbot is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members +# Origins: buildbot.steps.master.py +# buildbot.steps.shell.py +# Modifyed by Gentoo Authors. +# Copyright 2021 Gentoo Authors + +import os +import pprint +import re + +from twisted.internet import defer +from twisted.internet import error +from twisted.internet import reactor +from twisted.internet.protocol import ProcessProtocol +from twisted.python import runtime +from twisted.python.versions import Version + +from buildbot.process import buildstep +from buildbot.process import logobserver +from buildbot.process.results import FAILURE +from buildbot.process.results import SUCCESS +from buildbot.util import deferwaiter + + +class MasterSetPropertyFromCommand(buildstep.ShellMixin, buildstep.BuildStep): + + """ + Run a shell command locally - on the buildmaster. The shell command + COMMAND is specified just as for a RemoteShellCommand. Note that extra + logfiles are not supported. + """ + name = "Mastersetproperty" + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + renderables = ['command', 'env', 'property'] + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, command, property=None, extract_fn=None, strip=True, + includeStdout=True, includeStderr=False, **kwargs): + self.env = kwargs.pop('env', None) + self.usePTY = kwargs.pop('usePTY', 0) + self.interruptSignal = kwargs.pop('interruptSignal', 'KILL') + self.logEnviron = kwargs.pop('logEnviron', True) + + self.property = property + self.extract_fn = extract_fn + self.strip = strip + self.includeStdout = includeStdout + self.includeStderr = includeStderr + + if not ((property is not None) ^ (extract_fn is not None)): + config.error( + "Exactly one of property and extract_fn must be set") + + super().__init__(**kwargs) + + self.command = command + self.masterWorkdir = self.workdir + self._deferwaiter = deferwaiter.DeferWaiter() + self._status_object = None + self.success = True + + if self.extract_fn: + self.includeStderr = True + + self.observer = logobserver.BufferLogObserver( + wantStdout=self.includeStdout, + wantStderr=self.includeStderr) + self.addLogObserver('stdio', self.observer) + + class LocalPP(ProcessProtocol): + + def __init__(self, step): + self.step = step + self._finish_d = defer.Deferred() + self.step._deferwaiter.add(self._finish_d) + + def outReceived(self, data): + self.step._deferwaiter.add(self.step.stdio_log.addStdout(data)) + + def errReceived(self, data): + self.step._deferwaiter.add(self.step.stdio_log.addStderr(data)) + + def processEnded(self, status_object): + if status_object.value.exitCode is not None: + msg = "exit status {}\n".format(status_object.value.exitCode) + self.step._deferwaiter.add(self.step.stdio_log.addHeader(msg)) + + if status_object.value.signal is not None: + msg = "signal {}\n".format(status_object.value.signal) + self.step._deferwaiter.add(self.step.stdio_log.addHeader(msg)) + + self.step._status_object = status_object + self._finish_d.callback(None) + + @defer.inlineCallbacks + def run(self): + # render properties + command = self.command + # set up argv + if isinstance(command, (str, bytes)): + if runtime.platformType == 'win32': + # allow %COMSPEC% to have args + argv = os.environ['COMSPEC'].split() + if '/c' not in argv: + argv += ['/c'] + argv += [command] + else: + # for posix, use /bin/sh. for other non-posix, well, doesn't + # hurt to try + argv = ['/bin/sh', '-c', command] + else: + if runtime.platformType == 'win32': + # allow %COMSPEC% to have args + argv = os.environ['COMSPEC'].split() + if '/c' not in argv: + argv += ['/c'] + argv += list(command) + else: + argv = command + + self.stdio_log = yield self.addLog("stdio") + + if isinstance(command, (str, bytes)): + yield self.stdio_log.addHeader(command.strip() + "\n\n") + else: + yield self.stdio_log.addHeader(" ".join(command) + "\n\n") + yield self.stdio_log.addHeader("** RUNNING ON BUILDMASTER **\n") + yield self.stdio_log.addHeader(" in dir {}\n".format(os.getcwd())) + yield self.stdio_log.addHeader(" argv: {}\n".format(argv)) + + if self.env is None: + env = os.environ + else: + assert isinstance(self.env, dict) + env = self.env + for key, v in self.env.items(): + if isinstance(v, list): + # Need to do os.pathsep translation. We could either do that + # by replacing all incoming ':'s with os.pathsep, or by + # accepting lists. I like lists better. + # If it's not a string, treat it as a sequence to be + # turned in to a string. + self.env[key] = os.pathsep.join(self.env[key]) + + # do substitution on variable values matching pattern: ${name} + p = re.compile(r'\${([0-9a-zA-Z_]*)}') + + def subst(match): + return os.environ.get(match.group(1), "") + newenv = {} + for key, v in env.items(): + if v is not None: + if not isinstance(v, (str, bytes)): + raise RuntimeError(("'env' values must be strings or " + "lists; key '{}' is incorrect").format(key)) + newenv[key] = p.sub(subst, env[key]) + env = newenv + + if self.logEnviron: + yield self.stdio_log.addHeader(" env: %r\n" % (env,)) + + # TODO add a timeout? + self.process = reactor.spawnProcess(self.LocalPP(self), argv[0], argv, + path=self.masterWorkdir, usePTY=self.usePTY, env=env) + + # self._deferwaiter will yield only after LocalPP finishes + + yield self._deferwaiter.wait() + status_value = self._status_object.value + if status_value.signal is not None: + self.descriptionDone = ["killed ({})".format(status_value.signal)] + self.success = False + elif status_value.exitCode != 0: + self.descriptionDone = ["failed ({})".format(status_value.exitCode)] + self.success = False + + + yield self.stdio_log.finish() + + property_changes = {} + + if self.property: + if not self.success: + return FAILURE + result = self.observer.getStdout() + if self.strip: + result = result.strip() + propname = self.property + self.setProperty(propname, result, "MastertSetPropertyFromCommand Step") + property_changes[propname] = result + else: + new_props = self.extract_fn(self._status_object.value.exitCode, + self.observer.getStdout(), + self.observer.getStderr()) + for k, v in new_props.items(): + self.setProperty(k, v, "MasterSetPropertyFromCommand Step") + property_changes = new_props + + props_set = ["{}: {}".format(k, repr(v)) + for k, v in sorted(property_changes.items())] + yield self.addCompleteLog('property changes', "\n".join(props_set)) + + if len(property_changes) > 1: + self.descriptionDone = '{} properties set'.format(len(property_changes)) + elif len(property_changes) == 1: + self.descriptionDone = 'property \'{}\' set'.format(list(property_changes)[0]) + if not self.success: + return FAILURE + return SUCCESS + + def interrupt(self, reason): + try: + self.process.signalProcess(self.interruptSignal) + except KeyError: # Process not started yet + pass + except error.ProcessExitedAlready: + pass + super().interrupt(reason) diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py index 4fbe141..6d8388a 100644 --- a/buildbot_gentoo_ci/steps/portage.py +++ b/buildbot_gentoo_ci/steps/portage.py @@ -2,6 +2,14 @@ # Distributed under the terms of the GNU General Public License v2 import os +import io + +from portage import config as portage_config +from portage import auxdbkeys +from portage import _encodings +from portage import _unicode_encode +from portage import _parse_eapi_ebuild_head, eapi_is_supported +from portage.versions import cpv_getversion, pkgsplit, catpkgsplit from twisted.internet import defer from twisted.python import log @@ -11,6 +19,8 @@ from buildbot.process.results import SUCCESS from buildbot.process.results import FAILURE from buildbot.plugins import steps +from buildbot_gentoo_ci.steps import master as master_steps + @defer.inlineCallbacks def WriteTextToFile(path, text_list): separator = '\n' @@ -20,6 +30,33 @@ def WriteTextToFile(path, text_list): yield f.write(separator) yield f.close +def PersOutputOfEbuildSH(rc, stdout, stderr): + metadata = None + metadata_lines = stdout.splitlines() + metadata_valid = True + NoSplit = [] + NoSplit.append('DESCRIPTION') + if len(auxdbkeys) != len(metadata_lines): + # Don't trust bash's returncode if the + # number of lines is incorrect. + return { + 'auxdb' : metadata + } + else: + metadata_tmp = dict(zip(auxdbkeys, metadata_lines)) + metadata = {} + for k, v in metadata_tmp.items(): + if v == '': + metadata[k] = None + else: + if ' ' in v and k not in NoSplit: + metadata[k] = v.split(' ') + else: + metadata[k] = v + return { + 'auxdb' : metadata + } + class SetMakeProfile(BuildStep): name = 'SetMakeProfile' @@ -316,7 +353,7 @@ class SetMakeProfileLocal(BuildStep): @defer.inlineCallbacks def run(self): - parent_path = yield os.path.join('portage', 'make.profile', 'parent') + parent_path = yield os.path.join('etc','portage', 'make.profile', 'parent') if os.path.isfile(parent_path): return SUCCESS self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] @@ -325,7 +362,7 @@ class SetMakeProfileLocal(BuildStep): makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'], 'make.profile') for makeprofile in makeprofiles_data: makeprofile_path = yield os.path.join(self.repository_basedir, self.getProperty("profile_repository_data")['name'], 'profiles', makeprofile['value'], '') - makeprofiles_paths.append('../../' + makeprofile_path) + makeprofiles_paths.append('../../../' + makeprofile_path) yield WriteTextToFile(parent_path, makeprofiles_paths) return SUCCESS @@ -343,7 +380,7 @@ class SetReposConfLocal(BuildStep): @defer.inlineCallbacks def run(self): - repos_conf_path = yield os.path.join('portage', 'repos.conf') + repos_conf_path = yield os.path.join('etc', 'portage', 'repos.conf') repos_conf_default_path = yield os.path.join(repos_conf_path, 'default.conf') self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] self.repository_basedir = self.gentooci.config.project['repository_basedir'] @@ -360,7 +397,7 @@ class SetReposConfLocal(BuildStep): yield WriteTextToFile(repos_conf_default_path, default_conf) repos_conf_repository_path = yield os.path.join(repos_conf_path, self.getProperty("repository_data")['name'] + '.conf') if not os.path.isfile(repos_conf_repository_path): - repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name']) + repository_path = yield os.path.join(self.getProperty("builddir"), self.repository_basedir, self.getProperty("repository_data")['name']) repository_conf = [] repository_conf.append('[' + self.getProperty("repository_data")['name'] + ']') repository_conf.append('location = ' + repository_path) @@ -384,7 +421,7 @@ class SetMakeConfLocal(BuildStep): @defer.inlineCallbacks def run(self): - make_conf_path = yield os.path.join('portage', 'make.conf') + make_conf_path = yield os.path.join('etc', 'portage', 'make.conf') if os.path.isfile(make_conf_path): return SUCCESS makeconf_list = [] @@ -398,3 +435,86 @@ class SetMakeConfLocal(BuildStep): makeconf_list.append('FEATURES=""') yield WriteTextToFile(make_conf_path, makeconf_list) return SUCCESS + +class SetEnvForEbuildSH(BuildStep): + + name = 'SetEnvForEbuildSH' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def getEapiFromFile(self): + with io.open(_unicode_encode(self.getProperty("ebuild_file"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as f: + _eapi, _eapi_lineno = _parse_eapi_ebuild_head(f) + + return _eapi + + @defer.inlineCallbacks + def run(self): + addStepEbuildSH = [] + ebuild_commands = [] + ebuild_env = {} + config_root = yield os.path.join(self.getProperty("builddir"), '') + mysettings = yield portage_config(config_root = config_root) + + #Get EAPI from file and add it to env + eapi = yield self.getEapiFromFile() + print(eapi) + if eapi is None or not eapi_is_supported(eapi): + print('invalid eapi') + eapi = '0' + print(eapi_is_supported(eapi)) + ebuild_env['EAPI'] = eapi + + #FIXME: check manifest on ebuild_file + + #Setup ENV + category = yield catpkgsplit(self.getProperty("cpv"))[0] + package = yield catpkgsplit(self.getProperty("cpv"))[1] + version = yield catpkgsplit(self.getProperty("cpv"))[2] + revision = yield catpkgsplit(self.getProperty("cpv"))[3] + portage_bin_path = mysettings["PORTAGE_BIN_PATH"] + ebuild_sh_path = yield os.path.join(portage_bin_path, 'ebuild.sh') + #ebuild_env['PORTAGE_DEBUG'] = '1' + ebuild_env['EBUILD_PHASE'] = 'depend' + ebuild_env['CATEGORY'] = category + ebuild_env['P'] = package + '-' + version + ebuild_env['PN'] = package + ebuild_env['PR'] = revision + ebuild_env['PV'] = version + if revision == 'r0': + ebuild_env['PF'] = ebuild_env['P'] + ebuild_env['PVR'] = version + else: + ebuild_env['PF'] = ebuild_env['P'] + '-' + revision + ebuild_env['PVR'] = version + '-' + revision + ebuild_env['PORTAGE_BIN_PATH'] = portage_bin_path + ebuild_env['EBUILD'] = self.getProperty("ebuild_file") + ebuild_env['PORTAGE_PIPE_FD'] = '1' + ebuild_env['WORKDIR'] = yield os.path.join(mysettings["PORTAGE_TMPDIR"], 'portage', category, ebuild_env['PF'], 'work') + ebuild_env['PORTAGE_ECLASS_LOCATIONS'] = self.getProperty("repository_path") + + #FIXME: use sandbox if in FEATURES + ebuild_commands.append(ebuild_sh_path) + ebuild_commands.append('depend') + + addStepEbuildSH.append(master_steps.MasterSetPropertyFromCommand( + name = 'RunEbuildSH', + haltOnFailure = True, + flunkOnFailure = True, + command=ebuild_commands, + env=ebuild_env, + workdir=self.getProperty("builddir"), + strip=True, + extract_fn=PersOutputOfEbuildSH + )) + yield self.build.addStepsAfterCurrentStep(addStepEbuildSH) + return SUCCESS diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py index d47a253..143a758 100644 --- a/buildbot_gentoo_ci/steps/version.py +++ b/buildbot_gentoo_ci/steps/version.py @@ -7,10 +7,7 @@ import git from portage.xml.metadata import MetaDataXML from portage.checksum import perform_checksum -from portage.versions import cpv_getversion, pkgsplit -from portage import auxdbkeys -from portage import config as portage_config -from portage import portdbapi +from portage.versions import cpv_getversion, pkgsplit, catpkgsplit from twisted.internet import defer from twisted.python import log @@ -20,6 +17,8 @@ from buildbot.process.results import SUCCESS from buildbot.process.results import FAILURE from buildbot.plugins import steps +from buildbot_gentoo_ci.steps import portage as portage_steps + class GetVData(BuildStep): name = 'GetVData' @@ -34,6 +33,8 @@ class GetVData(BuildStep): @defer.inlineCallbacks def run(self): + # set cwd to builddir + yield os.chdir(self.getProperty("builddir")) self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] self.version = yield cpv_getversion(self.getProperty("cpv")) print(self.version) @@ -73,33 +74,7 @@ class AddVersion(BuildStep): self.setProperty("version_data", self.version_data, 'version_data') return SUCCESS -class GetAuxMetadata(BuildStep): - - name = 'GetAuxMetadata' - description = 'Running' - descriptionDone = 'Ran' - descriptionSuffix = None - haltOnFailure = True - flunkOnFailure = True - def __init__(self, **kwargs): - super().__init__(**kwargs) - - @defer.inlineCallbacks - def run(self): - self.mysettings = yield portage_config(config_root = self.getProperty("config_root")) - self.myportdb = yield portdbapi(mysettings=self.mysettings) - try: - auxdb_list = yield self.myportdb.aux_get(self.getProperty("cpv"), auxdbkeys, myrepo=self.getProperty("repository_data")['name']) - except: - print("Failed to get aux data for %s" % self.getProperty("cpv")) - yield self.myportdb.close_caches() - yield portdbapi.portdbapi_instances.remove(self.myportdb) - return FAILURE - self.setProperty('aux_metadata', auxdb_list, 'aux_metadata') - yield self.myportdb.close_caches() - yield portdbapi.portdbapi_instances.remove(self.myportdb) - return SUCCESS class GetCommitdata(BuildStep): @@ -113,19 +88,10 @@ class GetCommitdata(BuildStep): def __init__(self, **kwargs): super().__init__(**kwargs) - @defer.inlineCallbacks + #@defer.inlineCallbacks def run(self): - #FIXME: Could be a better way to get the log - git_log_ebuild = '' - g = git.Git(self.getProperty("repository_path")) - index = 1 - git_log_dict = {} - git_log = yield g.log('-n 1', self.getProperty("ebuild_file")) - print(git_log) - for line in git_log.splitlines(): - git_log_dict[index] = line - index = index + 1 - self.setProperty('commit_id', re.sub('commit ', '', git_log_dict[1]), 'commit_id') + print(self.getProperty("revision_data")) + self.setProperty('commit_id', self.getProperty("revision_data")['revision'], 'commit_id') return SUCCESS class AddVersionKeyword(BuildStep): @@ -158,7 +124,10 @@ class AddVersionKeyword(BuildStep): def run(self): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] self.version_keyword_dict = {} - for keyword in self.getProperty("aux_metadata")[8].split(): + auxdb = self.getProperty("auxdb")['KEYWORDS'] + if auxdb is None: + auxdb = [] + for keyword in auxdb: status = 'stable' if keyword[0] in ["~"]: keyword = keyword[1:] @@ -178,6 +147,8 @@ class AddVersionKeyword(BuildStep): version_keyword_data['keyword_id'], version_keyword_data['status']) self.version_keyword_dict[keyword] = version_keyword_data + if self.version_keyword_dict == {}: + self.version_keyword_dict = None self.setProperty('version_keyword_dict', self.version_keyword_dict, 'version_keyword_dict') return SUCCESS @@ -197,7 +168,7 @@ class CheckPathHash(BuildStep): def run(self): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] self.repository_basedir = self.gentooci.config.project['repository_basedir'] - self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'] + '.git') + self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name']) self.cp_path = yield pkgsplit(self.getProperty("cpv"))[0] self.file_name = yield self.getProperty("package_data")['name'] + '-' + self.getProperty("version") + '.ebuild' self.ebuild_file = yield os.path.join(self.repository_path, self.cp_path, self.file_name) @@ -274,14 +245,17 @@ class CheckV(BuildStep): self.old_version_data = self.getProperty("old_version_data") self.ebuild_file = self.getProperty("ebuild_file") addStepVData = [] + print(self.ebuild_file) + print(self.old_version_data) + print(self.getProperty("ebuild_file_hash")) if self.getProperty("ebuild_file") is None and self.getProperty("old_version_data") is not None: addStepVData.append(TriggerBuildCheck()) addStepVData.append(DeleteOldVersion()) if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is not None: if self.getProperty("ebuild_file_hash") != self.getProperty("old_version_data")['file_hash']: addStepVData.append(GetCommitdata()) + addStepVData.append(portage_steps.SetEnvForEbuildSH()) addStepVData.append(AddVersion()) - addStepVData.append(GetAuxMetadata()) addStepVData.append(AddVersionKeyword()) addStepVData.append(TriggerBuildCheck()) addStepVData.append(DeleteOldVersion()) @@ -289,8 +263,8 @@ class CheckV(BuildStep): return SUCCESS if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is None: addStepVData.append(GetCommitdata()) + addStepVData.append(portage_steps.SetEnvForEbuildSH()) addStepVData.append(AddVersion()) - addStepVData.append(GetAuxMetadata()) addStepVData.append(AddVersionKeyword()) addStepVData.append(TriggerBuildCheck()) yield self.build.addStepsAfterCurrentStep(addStepVData)