KojiPkgSpec is a more flexible way to describe what packages should be downloaded from a Koji server.
KojiClient supersedes KojiDownloader adding support for KojiPkgSpec. KojiInstaller is also revamped to support KojiPkgSpec and KojiClient. Signed-off-by: Cleber Rosa <cr...@redhat.com> --- client/tests/kvm/installer.py | 63 ++++- client/tests/kvm/kvm_utils.py | 671 ++++++++++++++++++++++++++++++++++------- 2 files changed, 618 insertions(+), 116 deletions(-) diff --git a/client/tests/kvm/installer.py b/client/tests/kvm/installer.py index 6b2a6fe..c357f0d 100644 --- a/client/tests/kvm/installer.py +++ b/client/tests/kvm/installer.py @@ -392,7 +392,9 @@ class YumInstaller(BaseInstaller): class KojiInstaller(YumInstaller): """ Class that handles installing KVM from the fedora build service, koji. - It uses yum to install and remove packages. + + It uses yum to install and remove packages. Packages are specified + according to the syntax defined in the PkgSpec class. """ load_stock_modules = True def set_install_params(self, test, params): @@ -403,27 +405,37 @@ class KojiInstaller(YumInstaller): @param params: Dictionary with test arguments """ super(KojiInstaller, self).set_install_params(test, params) - default_koji_cmd = '/usr/bin/koji' - default_src_pkg = 'qemu' - self.src_pkg = params.get("src_pkg", default_src_pkg) self.tag = params.get("koji_tag", None) - self.build = params.get("koji_build", None) - self.koji_cmd = params.get("koji_cmd", default_koji_cmd) + self.koji_cmd = params.get("koji_cmd", None) + if self.tag is not None: + kvm_utils.set_default_koji_tag(self.tag) + self.koji_pkgs = eval(params.get("koji_pkgs", "[]")) def _get_packages(self): """ Downloads the specific arch RPMs for the specific build name. """ - downloader = kvm_utils.KojiDownloader(cmd=self.koji_cmd) - downloader.get(src_package=self.src_pkg, tag=self.tag, - build=self.build, dst_dir=self.srcdir) + koji_client = kvm_utils.KojiClient(cmd=self.koji_cmd) + for pkg_text in self.koji_pkgs: + pkg = kvm_utils.KojiPkgSpec(pkg_text) + if pkg.is_valid(): + koji_client.get_pkgs(pkg, dst_dir=self.srcdir) + else: + logging.error('Package specification (%s) is invalid: %s' % \ + (pkg, pkg.describe_invalid())) + + + def _clean_previous_installs(self): + kill_qemu_processes() + removable_packages = " ".join(self._get_rpm_names()) + utils.system("yum -y remove %s" % removable_packages) def install(self): - super(KojiInstaller, self)._clean_previous_installs() + self._clean_previous_installs() self._get_packages() - super(KojiInstaller, self)._install_packages() + self._install_packages() self.install_unittests() create_symlinks(test_bindir=self.test_bindir, bin_list=self.qemu_bin_paths, @@ -433,6 +445,35 @@ class KojiInstaller(YumInstaller): save_build(self.srcdir, self.results_dir) + def _get_rpm_names(self): + all_rpm_names = [] + koji_client = kvm_utils.KojiClient(cmd=self.koji_cmd) + for pkg_text in self.koji_pkgs: + pkg = kvm_utils.KojiPkgSpec(pkg_text) + rpm_names = koji_client.get_pkg_rpm_names(pkg) + all_rpm_names += rpm_names + return all_rpm_names + + + def _get_rpm_file_names(self): + all_rpm_file_names = [] + koji_client = kvm_utils.KojiClient(cmd=self.koji_cmd) + for pkg_text in self.koji_pkgs: + pkg = kvm_utils.KojiPkgSpec(pkg_text) + rpm_file_names = koji_client.get_pkg_rpm_file_names(pkg) + all_rpm_file_names += rpm_file_names + return all_rpm_file_names + + + def _install_packages(self): + """ + Install all downloaded packages. + """ + os.chdir(self.srcdir) + rpm_file_names = " ".join(self._get_rpm_file_names()) + utils.system("yum --nogpgcheck -y localinstall %s" % rpm_file_names) + + class SourceDirInstaller(BaseInstaller): """ Class that handles building/installing KVM directly from a tarball or diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py index dba776d..fb015e2 100644 --- a/client/tests/kvm/kvm_utils.py +++ b/client/tests/kvm/kvm_utils.py @@ -1577,142 +1577,603 @@ class PciAssignable(object): return -class KojiDownloader(object): +class KojiClient(object): """ - Stablish a connection with the build system, either koji or brew. + Stablishes a connection with the build system, either koji or brew. - This class provides a convenience methods to retrieve packages hosted on - the build system. + This class provides convenience methods to retrieve information on packages + and the packages themselves hosted on the build system. Packages should be + specified in the KojiPgkSpec syntax. """ - def __init__(self, cmd): + + CMD_LOOKUP_ORDER = ['/usr/bin/brew', '/usr/bin/koji' ] + + CONFIG_MAP = {'/usr/bin/brew': '/etc/brewkoji.conf', + '/usr/bin/koji': '/etc/koji.conf'} + + + def __init__(self, cmd=None): """ Verifies whether the system has koji or brew installed, then loads the configuration file that will be used to download the files. - @param cmd: Command name, either 'brew' or 'koji'. It is important - to figure out the appropriate configuration used by the - downloader. - @param dst_dir: Destination dir for the packages. + @type cmd: string + @param cmd: Optional command name, either 'brew' or 'koji'. If not + set, get_default_command() is used and to look for + one of them. + @raise: ValueError """ if not KOJI_INSTALLED: raise ValueError('No koji/brew installed on the machine') - if os.path.isfile(cmd): - koji_cmd = cmd + # Instance variables used by many methods + self.command = None + self.config = None + self.config_options = {} + self.session = None + + # Set koji command or get default + if cmd is None: + self.command = self.get_default_command() else: - koji_cmd = os_dep.command(cmd) + self.command = cmd - logging.debug("Found %s as the buildsystem interface", koji_cmd) + # Check koji command + if not self.is_command_valid(): + raise ValueError('Koji command "%s" is not valid' % self.command) + + # Assuming command is valid, set configuration file and read it + self.config = self.CONFIG_MAP[self.command] + self.read_config() + + # Setup koji session + server_url = self.config_options['server'] + session_options = self.get_session_options() + self.session = koji.ClientSession(server_url, + session_options) - config_map = {'/usr/bin/koji': '/etc/koji.conf', - '/usr/bin/brew': '/etc/brewkoji.conf'} - try: - config_file = config_map[koji_cmd] - except IndexError: - raise ValueError('Could not find config file for %s' % koji_cmd) - - base_name = os.path.basename(koji_cmd) - if os.access(config_file, os.F_OK): - f = open(config_file) - config = ConfigParser.ConfigParser() - config.readfp(f) - f.close() - else: - raise IOError('Configuration file %s missing or with wrong ' - 'permissions' % config_file) - - if config.has_section(base_name): - self.koji_options = {} - session_options = {} - server = None - for name, value in config.items(base_name): - if name in ('user', 'password', 'debug_xmlrpc', 'debug'): - session_options[name] = value - self.koji_options[name] = value - self.session = koji.ClientSession(self.koji_options['server'], - session_options) - else: - raise ValueError('Koji config file %s does not have a %s ' - 'session' % (config_file, base_name)) + def read_config(self, check_is_valid=True): + ''' + Reads options from the Koji configuration file + By default it checks if the koji configuration is valid + + @type check_valid: boolean + @param check_valid: whether to include a check on the configuration + @raises: ValueError + @returns: None + ''' + if check_is_valid: + if not self.is_config_valid(): + raise ValueError('Koji config "%s" is not valid' % self.config) + + config = ConfigParser.ConfigParser() + config.read(self.config) + + basename = os.path.basename(self.command) + for name, value in config.items(basename): + self.config_options[name] = value + + + def get_session_options(self): + ''' + Filter only options necessary for setting up a cobbler client session + + @returns: only the options used for session setup + ''' + session_options = {} + for name, value in self.config_options.items(): + if name in ('user', 'password', 'debug_xmlrpc', 'debug'): + session_options[name] = value + return session_options + + + def is_command_valid(self): + ''' + Checks if the currently set koji command is valid + + @returns: True or False + ''' + koji_command_ok = True + + if not os.path.isfile(self.command): + logging.error('Koji command "%s" is not a regular file' % \ + self.command) + koji_command_ok = False + + if not os.access(self.command, os.X_OK): + logging.warn('Koji command "%s" is not executable: this is ' + 'not fatal but indicates an unexpected situation' % \ + self.command) + + if not self.command in self.CONFIG_MAP.keys(): + logging.error('Koji command "%s" does not have a configuration ' \ + 'file associated to it') % self.command + koji_command_ok = False + + return koji_command_ok + + + def is_config_valid(self): + ''' + Checks if the currently set koji configuration is valid + + @returns: True or False + ''' + koji_config_ok = True + + if not os.path.isfile(self.config): + logging.error('Koji config "%s" is not a regular file' % \ + self.config) + koji_config_ok = False + + if not os.access(self.config, os.R_OK): + logging.error('Koji config "%s" is not readable' % \ + self.config) + koji_config_ok = False + + config = ConfigParser.ConfigParser() + config.read(self.config) + basename = os.path.basename(self.command) + if not config.has_section(basename): + logging.error('Koji configuration file "%s" does not have a ' + 'section "%s", named after the base name of the ' + 'currently set koji command "%s"' % + (self.config, + basename, + self.command)) + koji_config_ok = False + + return koji_config_ok + + + def get_default_command(self): + ''' + Looks up for koji or brew "binaries" on the system + + Systems with plain koji usually don't have a brew cmd, while systems + with koji, have *both* koji and brew utilities. So we look for brew + first, and if found, we consider that the system is configured for + brew. If not, we consider this is a system with plain koji. + + @returns: either koji or brew command line executable path, or None + ''' + koji_command = None + for command in self.CMD_LOOKUP_ORDER: + if os.path.isfile(command): + koji_command = command + break + else: + koji_command_basename = os.path.basename(koji_command) + try: + koji_command = os_dep.command(koji_command_basename) + break + except ValueError: + pass + return koji_command + + + def get_pkg_info(self, pkg): + ''' + Returns information from Koji on the package + + @type pkg: KojiPkgSpec + @param pkg: information about the package, as a KojiPkgSpec instance + + @returns: information from Koji about the specified package + ''' + info = {} + if pkg.build is not None: + info = self.session.getBuild(int(pkg.build)) + elif pkg.tag is not None and pkg.package is not None: + builds = self.session.listTagged(pkg.tag, + latest=True, + inherit=True, + package=pkg.package) + if builds: + info = builds[0] + return info + + + def is_pkg_valid(self, pkg): + ''' + Checks if this package is altogether valid on Koji + + This verifies if the build or tag specified in the package + specification actually exist on the Koji server + + @returns: True or False + ''' + valid = True + if not self.is_pkg_spec_build_valid(pkg): + valid = False + if not self.is_pkg_spec_tag_valid(pkg): + valid = False + return valid + + + def is_pkg_spec_build_valid(self, pkg): + ''' + Checks if build is valid on Koji + + @param pkg: a Pkg instance + ''' + if pkg.build is not None: + info = self.session.getBuild(int(pkg.build)) + if info: + return True + return False + + + def is_pkg_spec_tag_valid(self, pkg): + ''' + Checks if tag is valid on Koji + + @type pkg: KojiPkgSpec + @param pkg: a package specification + ''' + if pkg.tag is not None: + tag = self.session.getTag(pkg.tag) + if tag: + return True + return False - def get(self, src_package, dst_dir, rfilter=None, tag=None, build=None, - arch=None): - """ - Download a list of packages from the build system. - - This will download all packages originated from source package [package] - with given [tag] or [build] for the architecture reported by the - machine. - - @param src_package: Source package name. - @param dst_dir: Destination directory for the downloaded packages. - @param rfilter: Regexp filter, only download the packages that match - that particular filter. - @param tag: Build system tag. - @param build: Build system ID. - @param arch: Package arch. Useful when you want to download noarch - packages. - - @return: List of paths with the downloaded rpm packages. - """ - if build and build.isdigit(): - build = int(build) - - if tag and build: - logging.info("Both tag and build parameters provided, ignoring tag " - "parameter...") - - if not tag and not build: - raise ValueError("Koji install selected but neither koji_tag " - "nor koji_build parameters provided. Please " - "provide an appropriate tag or build name.") - - if not build: - builds = self.session.listTagged(tag, latest=True, inherit=True, - package=src_package) - if not builds: - raise ValueError("Tag %s has no builds of %s" % (tag, - src_package)) - info = builds[0] - else: - info = self.session.getBuild(build) - if info is None: - raise ValueError('No such brew/koji build: %s' % build) + def get_pkg_rpm_info(self, pkg, arch=None): + ''' + Returns a list of infomation on the RPM packages found on koji + @type pkg: KojiPkgSpec + @param pkg: a package specification + @type arch: string + @param arch: packages built for this architecture, but also including + architecture independent (noarch) packages + ''' + if arch is None: + arch = utils.get_arch() + rpms = [] + info = self.get_pkg_info(pkg) + if info: + rpms = self.session.listRPMs(buildID=info['id'], + arches=[arch, 'noarch']) + if pkg.subpackages: + rpms = [d for d in rpms if d['name'] in pkg.subpackages] + return rpms + + + def get_pkg_rpm_names(self, pkg, arch=None): + ''' + Gets the names for the RPM packages specified in pkg + + @type pkg: KojiPkgSpec + @param pkg: a package specification + @type arch: string + @param arch: packages built for this architecture, but also including + architecture independent (noarch) packages + ''' if arch is None: arch = utils.get_arch() + rpms = self.get_pkg_rpm_info(pkg, arch) + return [rpm['name'] for rpm in rpms] + - rpms = self.session.listRPMs(buildID=info['id'], - arches=arch) - if not rpms: - raise ValueError("No %s packages available for %s" % - arch, koji.buildLabel(info)) + def get_pkg_rpm_file_names(self, pkg, arch=None): + ''' + Gets the file names for the RPM packages specified in pkg - rpm_paths = [] + @type pkg: KojiPkgSpec + @param pkg: a package specification + @type arch: string + @param arch: packages built for this architecture, but also including + architecture independent (noarch) packages + ''' + if arch is None: + arch = utils.get_arch() + rpm_names = [] + rpms = self.get_pkg_rpm_info(pkg, arch) + for rpm in rpms: + arch_rpm_name = koji.pathinfo.rpm(rpm) + rpm_name = os.path.basename(arch_rpm_name) + rpm_names.append(rpm_name) + return rpm_names + + + def get_pkg_urls(self, pkg, arch=None): + ''' + Gets the urls for the packages specified in pkg + + @type pkg: KojiPkgSpec + @param pkg: a package specification + @type arch: string + @param arch: packages built for this architecture, but also including + architecture independent (noarch) packages + ''' + info = self.get_pkg_info(pkg) + rpms = self.get_pkg_rpm_info(pkg, arch) + rpm_urls = [] for rpm in rpms: rpm_name = koji.pathinfo.rpm(rpm) - url = ("%s/%s/%s/%s/%s" % (self.koji_options['pkgurl'], + url = ("%s/%s/%s/%s/%s" % (self.config_options['pkgurl'], info['package_name'], info['version'], info['release'], rpm_name)) - if rfilter: - filter_regexp = re.compile(rfilter, re.IGNORECASE) - if filter_regexp.match(os.path.basename(rpm_name)): - download = True - else: - download = False + rpm_urls.append(url) + return rpm_urls + + + def get_pkgs(self, pkg, dst_dir, arch=None): + ''' + Download the packages + + @type pkg: KojiPkgSpec + @param pkg: a package specification + @type dst_dir: string + @param dst_dir: the destination directory, where the downloaded + packages will be saved on + @type arch: string + @param arch: packages built for this architecture, but also including + architecture independent (noarch) packages + ''' + rpm_urls = self.get_pkg_urls(pkg, arch) + for url in rpm_urls: + utils.get_file(url, + os.path.join(dst_dir, os.path.basename(url))) + + +DEFAULT_KOJI_TAG = None +def set_default_koji_tag(tag): + ''' + Sets the default tag that will be used + ''' + global DEFAULT_KOJI_TAG + DEFAULT_KOJI_TAG = tag + +def get_default_koji_tag(): + return DEFAULT_KOJI_TAG + + +class KojiPkgSpec: + ''' + A package specification syntax parser for Koji + + This holds information on either tag or build, and packages to be fetched + from koji and possibly installed (features external do this class). + + New objects can be created either by providing information in the textual + format or by using the actual parameters for tag, build, package and sub- + packages. The textual format is useful for command line interfaces and + configuration files, while using parameters is better for using this in + a programatic fashion. + + The following sets of examples are interchangeable. Specifying all packages + part of build number 1000: + + >>> from kvm_utils import KojiPkgSpec + >>> pkg = KojiPkgSpec('1000') + + >>> pkg = KojiPkgSpec(build=1000) + + Specifying only a subset of packages of build number 1000: + + >>> pkg = KojiPkgSpec('1000:kernel,kernel-devel') + + >>> pkg = KojiPkgSpec(build=1000, + subpackages=['kernel', 'kernel-devel']) + + Specifying the latest build for the 'kernel' package tagged with 'dist-f14': + + >>> pkg = KojiPkgSpec('dist-f14:kernel') + + >>> pkg = KojiPkgSpec(tag='dist-f14', package='kernel') + + Specifying the 'kernel' package using the default tag: + + >>> kvm_utils.set_default_koji_tag('dist-f14') + >>> pkg = KojiPkgSpec('kernel') + + >>> pkg = KojiPkgSpec(package='kernel') + + Specifying the 'kernel' package using the default tag: + + >>> kvm_utils.set_default_koji_tag('dist-f14') + >>> pkg = KojiPkgSpec('kernel') + + >>> pkg = KojiPkgSpec(package='kernel') + + If you do not specify a default tag, and give a package name without an + explicit tag, your package specification is considered invalid: + + >>> print kvm_utils.get_default_koji_tag() + None + >>> print kvm_utils.KojiPkgSpec('kernel').is_valid() + False + + >>> print kvm_utils.KojiPkgSpec(package='kernel').is_valid() + False + ''' + + SEP = ':' + + def __init__(self, text='', tag=None, build=None, + package=None, subpackages=[]): + ''' + Instantiates a new KojiPkgSpec object + + @type text: string + @param text: a textual representation of a package on Koji that + will be parsed + @type tag: string + @param tag: a koji tag, example: Fedora-14-RELEASE + (see U{http://fedoraproject.org/wiki/Koji#Tags_and_Targets}) + @type build: number + @param build: a koji build, example: 1001 + (see U{http://fedoraproject.org/wiki/Koji#Koji_Architecture}) + @type package: string + @param package: a koji package, example: python + (see U{http://fedoraproject.org/wiki/Koji#Koji_Architecture}) + @type subpackages: list of strings + @param subpackages: a list of package names, usually a subset of + the RPM packages generated by a given build + ''' + + # Set to None to indicate 'not set' (and be able to use 'is') + self.tag = None + self.build = None + self.package = None + self.subpackages = [] + + self.default_tag = None + + # Textual representation takes precedence (most common use case) + if text: + self.parse(text) + else: + self.tag = tag + self.build = build + self.package = package + self.subpackages = subpackages + + # Set the default tag, if set, as a fallback + if not self.build and not self.tag: + default_tag = get_default_koji_tag() + if default_tag is not None: + self.tag = default_tag + + + def parse(self, text): + ''' + Parses a textual representation of a package specification + + @type text: string + @param text: textual representation of a package in koji + ''' + parts = text.count(self.SEP) + 1 + if parts == 1: + if text.isdigit(): + self.build = text + else: + self.package = text + elif parts == 2: + part1, part2 = text.split(self.SEP) + if part1.isdigit(): + self.build = part1 + self.subpackages = part2.split(',') + else: + self.tag = part1 + self.package = part2 + elif parts >= 3: + # Instead of erroring on more arguments, we simply ignore them + # This makes the parser suitable for future syntax additions, such + # as specifying the package architecture + part1, part2, part3 = text.split(self.SEP)[0:3] + self.tag = part1 + self.package = part2 + self.subpackages = part3.split(',') + + + def _is_invalid_neither_tag_or_build(self): + ''' + Checks if this package is invalid due to not having either a valid + tag or build set, that is, both are empty. + + @returns: True if this is invalid and False if it's valid + ''' + return (self.tag is None and self.build is None) + + + def _is_invalid_package_but_no_tag(self): + ''' + Checks if this package is invalid due to having a package name set + but tag or build set, that is, both are empty. + + @returns: True if this is invalid and False if it's valid + ''' + return (self.package and not self.tag) + + + def _is_invalid_subpackages_but_no_main_package(self): + ''' + Checks if this package is invalid due to having a tag set (this is Ok) + but specifying subpackage names without specifying the main package + name. + + Specifying subpackages without a main package name is only valid when + a build is used instead of a tag. + + @returns: True if this is invalid and False if it's valid + ''' + return (self.tag and self.subpackages and not self.package) + + + def is_valid(self): + ''' + Checks if this package specification is valid. + + Being valid means that it has enough and not conflicting information. + It does not validate that the packages specified actually existe on + the Koji server. + + @returns: True or False + ''' + if self._is_invalid_neither_tag_or_build(): + return False + elif self._is_invalid_package_but_no_tag(): + return False + elif self._is_invalid_subpackages_but_no_main_package(): + return False + + return True + + + def describe_invalid(self): + ''' + Describes why this is not valid, in a human friendly way + ''' + if self._is_invalid_neither_tag_or_build(): + return 'neither a tag or build are set, and of them should be set' + elif self._is_invalid_package_but_no_tag(): + return 'package name specified but no tag is set' + elif self._is_invalid_subpackages_but_no_main_package(): + return 'subpackages specified but no main package is set' + + return 'unkwown reason, seems to be valid' + + + def describe(self): + ''' + Describe this package specification, in a human friendly way + + @returns: package specification description + ''' + if self.is_valid(): + description = '' + if not self.subpackages: + description += 'all subpackages from %s ' % self.package + else: + description += 'only subpackage(s) %s from package %s ' % \ + (', '.join(self.subpackages), + self.package) + + if self.build: + description += 'from build %s' % self.build + elif self.tag: + description += 'tagged with %s' % self.tag else: - download = True + raise ValueError, 'neither build or tag is set' + + return description + else: + return 'Invalid package specification: %s' % \ + self.describe_invalid() - if download: - r = utils.get_file(url, - os.path.join(dst_dir, os.path.basename(url))) - rpm_paths.append(r) - return rpm_paths + def __repr__(self): + return "<KojiPkgSpec tag=%s build=%s pkg=%s subpkgs=%s>" \ + % (self.tag, + self.build, + self.package, + ", ".join(self.subpackages)) def umount(src, mount_point, type): -- 1.7.4 _______________________________________________ Autotest mailing list Autotest@test.kernel.org http://test.kernel.org/cgi-bin/mailman/listinfo/autotest