commit: 28828655da860324861af9adea0794a0a61196bf Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Wed Dec 10 03:19:17 2014 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Thu Dec 11 08:05:40 2014 +0000 URL: http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=28828655
Support @profile package set for bug #532224 Add support for a new @profile set which allows the profile to pull in additional packages that do not belong to the @system set. The motivation to have @profile separate from @system is that @system packages may have incomplete dependency specifications (due to long-standing Gentoo policy), and incomplete dependency specifications have deleterious effects on the ability of emerge --jobs to parallelize builds. So, unlike @system, packages added to @profile do not hurt emerge --jobs parallelization. Packages are added to the @profile set in the same way that they are added to the @system set, except that atoms in the @profile set are not preceded with a '*' character. Also, the @profile package set is only supported when 'profile-set' is listed in the layout.conf profile-formats field of the containing repository. X-Gentoo-Bug: 532224 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532224 Acked-by: Alexander Berntsen <bernalex <AT> gentoo.org> --- cnf/sets/portage.conf | 5 +- doc/config/sets.docbook | 21 +++- man/emerge.1 | 14 ++- man/portage.5 | 33 ++++-- pym/_emerge/actions.py | 14 ++- pym/portage/_sets/ProfilePackageSet.py | 33 ++++++ pym/portage/_sets/__init__.py | 2 +- .../package/ebuild/_config/LocationsManager.py | 2 +- pym/portage/package/ebuild/config.py | 4 +- pym/portage/repository/config.py | 2 +- .../tests/resolver/test_profile_package_set.py | 123 +++++++++++++++++++++ 11 files changed, 227 insertions(+), 26 deletions(-) diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf index 8ffcedc..cff7488 100644 --- a/cnf/sets/portage.conf +++ b/cnf/sets/portage.conf @@ -7,7 +7,10 @@ # Not much that could be changed for world, so better leave it alone [world] class = portage.sets.base.DummyPackageSet -packages = @selected @system +packages = @profile @selected @system + +[profile] +class = portage.sets.ProfilePackageSet.ProfilePackageSet [selected] class = portage.sets.files.WorldSelectedSet diff --git a/doc/config/sets.docbook b/doc/config/sets.docbook index d3aa147..749b775 100644 --- a/doc/config/sets.docbook +++ b/doc/config/sets.docbook @@ -286,7 +286,7 @@ <filename>packages</filename> files in the profile. <!-- TODO: Add reference to profile documentation regarding "packages" --> There is no reason to use this in a user configuration as it is already - confgured by default and doesn't support any options. + configured by default and doesn't support any options. </para> <sect3> @@ -296,7 +296,23 @@ </para> </sect3> </sect2> - + + <sect2 id='config-set-classes-ProfilePackageSet'> + <title>portage.sets.ProfilePackageSet.ProfilePackageSet</title> + <para> + This class implements the <parameter>profile</parameter> set, based on the + <filename>packages</filename> files in the profile. + There is no reason to use this in a user configuration as it is already + confgured by default and doesn't support any options. + </para> + <sect3> + <title>Single Set Configuration</title> + <para> + This class doesn't support any extra options. + </para> + </sect3> + </sect2> + <sect2 id='config-set-classes-SecuritySet' xreflabel='SecuritySet'> <title>portage.sets.security.SecuritySet</title> <para> @@ -601,6 +617,7 @@ </para> <itemizedlist> <listitem><para><varname>world</varname>: uses <classname>DummySet</classname></para></listitem> + <listitem><para><varname>profile</varname>: uses <classname>ProfilePackageSet</classname></para></listitem> <listitem><para><varname>selected</varname>: uses <classname>WorldSelectedSet</classname></para></listitem> <listitem><para><varname>system</varname>: uses <classname>PackagesSystemSet</classname></para></listitem> <listitem><para><varname>security</varname>: uses <classname>NewAffectedSet</classname> with default options</para></listitem> diff --git a/man/emerge.1 b/man/emerge.1 index f64fd1b..b36f59c 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -61,15 +61,17 @@ would like to query the owners of one or more files or directories. .TP .BR set A \fIset\fR is a convenient shorthand for a large group of -packages. Five sets are currently always available: \fBselected-packages\fR, -\fBselected-sets\fR, \fBselected\fR, \fBsystem\fR and \fBworld\fR. +packages. Six sets are currently always available: \fBselected-packages\fR, +\fBselected-sets\fR, \fBselected\fR, \fBsystem\fR, \fBprofile\fR, and \fBworld\fR. \fBselected-packages\fR contains the user-selected "world" packages that are listed in \fB/var/lib/portage/world\fR, while \fBselected-sets\fR contains the nested sets that may be listed in \fB/var/lib/portage/world_sets\fR. -\fBsystem\fR refers to a set of packages deemed necessary for your system -to run properly. \fBselected\fR encompasses both the \fBselected-packages\fR -and \fBselected-sets\fR sets, while \fBworld\fR encompasses the \fBselected\fR -and \fBsystem\fR sets. [See +\fBsystem\fR and \fBprofile\fR both refer to sets of packages deemed +necessary for your system to run properly (the differences between these +two sets are documented in \fBportage\fR(5)). +\fBselected\fR encompasses both the \fBselected-packages\fR +and \fBselected-sets\fR sets, while \fBworld\fR encompasses the \fBselected\fR, +\fBsystem\fR and \fBprofile\fR sets. [See \fBFILES\fR below for more information.] Other sets can exist depending on the current configuration. The default set configuration is located in the \fB/usr/share/portage/config/sets\fR directory. diff --git a/man/portage.5 b/man/portage.5 index 2fa699c..88cf3bb 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -1,4 +1,4 @@ -.TH "PORTAGE" "5" "Feb 2014" "Portage VERSION" "Portage" +.TH "PORTAGE" "5" "December 2014" "Portage VERSION" "Portage" .SH NAME portage \- the heart of Gentoo .SH "DESCRIPTION" @@ -331,14 +331,25 @@ Special USE flags which may be needed when bootstrapping from stage1 to stage2. .PD 1 .TP .BR packages -Provides the list of packages that compose the special \fIsystem\fR set. +Provides the list of packages that compose the \fI@system\fR and +\fI@profile\fR package sets. The motivation to have \fI@profile\fR +separate from \fI@system\fR is that \fI@system\fR packages may have +incomplete dependency specifications (due to long-standing Gentoo +policy), and incomplete dependency specifications have deleterious +effects on the ability of \fBemerge\fR to parallelize builds. So, +unlike \fI@system\fR, packages included in \fI@profile\fR do not +hurt \fBemerge\fR's ability to parallelize. .I Format: .nf \- comments begin with # (no inline comments) \- one DEPEND atom per line -\- packages to be added to the system set begin with a * -\- atoms without * only appear for legacy reasons +\- packages to be added to the @system set begin with a * +\- packages to be added to the @profile set do not begin with a * +\- packages may only be added to the @profile set if the containing + repository's layout.conf has 'profile-set' listed in the + profile-formats field. Otherwise, packages that do not begin with + '*' will simply be ignored for legacy reasons .fi .I Note: In a cascading profile setup, you can remove packages in children @@ -348,12 +359,14 @@ a '\-'. .I Example: .nf # i am a comment ! -# pull in a version of glibc less than 2.3 +# pull a version of glibc less than 2.3 into @system *<sys\-libs/glibc\-2.3 -# pull in any version of bash +# pull any version of bash into @system *app\-shells/bash -# pull in a version of readline earlier than 4.2 +# pull a version of readline earlier than 4.2 into @system *<sys\-libs/readline\-4.2 +# pull vim into @profile +app-editors/vim .fi .TP .BR packages.build @@ -1101,13 +1114,15 @@ The default setting for repoman's --echangelog option. The cache formats supported in the metadata tree. There is the old "pms" format and the newer/faster "md5-dict" format. Default is to detect dirs. .TP -.BR profile\-formats " = [pms|portage-1|portage-2|profile-bashrcs]" +.BR profile\-formats " = [pms|portage-1|portage-2|profile-bashrcs|profile-set]" Control functionality available to profiles in this repo such as which files may be dirs, or the syntax available in parent files. Use "portage-2" if you're unsure. The default is "portage-1-compat" mode which is meant to be compatible with old profiles, but is not allowed to be opted into directly. Setting profile-bashrcs will enable the per-profile bashrc mechanism -\fBpackage.bashrc\fR. +\fBpackage.bashrc\fR. Setting profile-set enables support for using the +profile \fBpackages\fR file to add atoms to the @profile package set. +See the profile \fBpackages\fR section for more information. .RE .RE diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index c7246a9..0ae2c16 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -650,7 +650,7 @@ def action_depclean(settings, trees, ldpath_mtimes, return rval set_atoms = {} - for k in ("system", "selected"): + for k in ("profile", "system", "selected"): try: set_atoms[k] = root_config.setconfig.getSetAtoms(k) except portage.exception.PackageSetNotFound: @@ -660,6 +660,8 @@ def action_depclean(settings, trees, ldpath_mtimes, print("Packages installed: " + str(len(vardb.cpv_all()))) print("Packages in world: %d" % len(set_atoms["selected"])) print("Packages in system: %d" % len(set_atoms["system"])) + if set_atoms["profile"]: + print("Packages in profile: %d" % len(set_atoms["profile"])) print("Required packages: "+str(req_pkg_count)) if "--pretend" in myopts: print("Number to remove: "+str(len(cleanlist))) @@ -693,20 +695,24 @@ def calc_depclean(settings, trees, ldpath_mtimes, system_set = psets["system"] set_atoms = {} - for k in ("system", "selected"): + for k in ("profile", "system", "selected"): try: set_atoms[k] = root_config.setconfig.getSetAtoms(k) except portage.exception.PackageSetNotFound: # A nested set could not be resolved, so ignore nested sets. set_atoms[k] = root_config.sets[k].getAtoms() - if not set_atoms["system"] or not set_atoms["selected"]: + if (not set_atoms["system"] or + not (set_atoms["selected"] or set_atoms["profile"])): if not set_atoms["system"]: writemsg_level("!!! You have no system list.\n", level=logging.ERROR, noiselevel=-1) - if not set_atoms["selected"]: + # Skip this warning if @profile is non-empty, in order to + # support using @profile as an alternative to @selected + # for building a stage 4. + if not (set_atoms["selected"] or set_atoms["profile"]): writemsg_level("!!! You have no world file.\n", level=logging.WARNING, noiselevel=-1) diff --git a/pym/portage/_sets/ProfilePackageSet.py b/pym/portage/_sets/ProfilePackageSet.py new file mode 100644 index 0000000..c2f5fee --- /dev/null +++ b/pym/portage/_sets/ProfilePackageSet.py @@ -0,0 +1,33 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage import os +from portage.util import grabfile_package, stack_lists +from portage._sets.base import PackageSet + +class ProfilePackageSet(PackageSet): + _operations = ["merge"] + + def __init__(self, profiles, debug=False): + super(ProfilePackageSet, self).__init__() + self._profiles = profiles + if profiles: + desc_profile = profiles[-1] + if desc_profile.user_config and len(profiles) > 1: + desc_profile = profiles[-2] + description = desc_profile.location + else: + description = None + self.description = "Profile packages for profile %s" % description + + def load(self): + self._setAtoms(x for x in stack_lists( + [grabfile_package(os.path.join(y.location, "packages"), + verify_eapi=True) for y in self._profiles + if "profile-set" in y.profile_formats], + incremental=1) if x[:1] != "*") + + def singleBuilder(self, options, settings, trees): + return ProfilePackageSet( + settings._locations_manager.profiles_complex) + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/_sets/__init__.py b/pym/portage/_sets/__init__.py index 5e4a9dc..47be418 100644 --- a/pym/portage/_sets/__init__.py +++ b/pym/portage/_sets/__init__.py @@ -116,7 +116,7 @@ class SetConfig(object): parser.remove_section("world") parser.add_section("world") parser.set("world", "class", "portage.sets.base.DummyPackageSet") - parser.set("world", "packages", "@selected @system") + parser.set("world", "packages", "@profile @selected @system") parser.remove_section("selected") parser.add_section("selected") diff --git a/pym/portage/package/ebuild/_config/LocationsManager.py b/pym/portage/package/ebuild/_config/LocationsManager.py index 6280a42..6641092 100644 --- a/pym/portage/package/ebuild/_config/LocationsManager.py +++ b/pym/portage/package/ebuild/_config/LocationsManager.py @@ -133,7 +133,7 @@ class LocationsManager(object): self.profiles.append(custom_prof) self.profiles_complex.append( _profile_node(custom_prof, True, True, - ('profile-bashrcs',))) + ('profile-bashrcs', 'profile-set'))) del custom_prof self.profiles = tuple(self.profiles) diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index c5c727b..65de93e 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -512,7 +512,6 @@ class config(object): if v is not None: portdir_sync = v - known_repos = frozenset(known_repos) self["PORTDIR"] = portdir self["PORTDIR_OVERLAY"] = portdir_overlay if portdir_sync: @@ -523,6 +522,9 @@ class config(object): else: self.repositories = repositories + known_repos.extend(repo.location for repo in self.repositories) + known_repos = frozenset(known_repos) + self['PORTAGE_REPOSITORIES'] = self.repositories.config_string() self.backup_changes('PORTAGE_REPOSITORIES') diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index f45684b..9096d73 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -41,7 +41,7 @@ if sys.hexversion >= 0x3000000: _invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]') _valid_profile_formats = frozenset( - ['pms', 'portage-1', 'portage-2', 'profile-bashrcs']) + ['pms', 'portage-1', 'portage-2', 'profile-bashrcs', 'profile-set']) _portage1_profiles_allow_directories = frozenset( ["portage-1-compat", "portage-1", 'portage-2']) diff --git a/pym/portage/tests/resolver/test_profile_package_set.py b/pym/portage/tests/resolver/test_profile_package_set.py new file mode 100644 index 0000000..88a2a82 --- /dev/null +++ b/pym/portage/tests/resolver/test_profile_package_set.py @@ -0,0 +1,123 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import unicode_literals + +import io + +from portage import os, _encodings +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ( + ResolverPlayground, ResolverPlaygroundTestCase) +from portage.util import ensure_dirs + +class ProfilePackageSetTestCase(TestCase): + + def testProfilePackageSet(self): + + repo_configs = { + "test_repo": { + "layout.conf": ("profile-formats = profile-set",), + } + } + + profiles = ( + ( + 'default/linux', + { + "eapi": ("5",), + "packages": ( + "*sys-libs/A", + "app-misc/A", + "app-misc/B", + "app-misc/C", + ), + } + ), + ( + 'default/linux/x86', + { + "eapi": ("5",), + "packages": ( + "-app-misc/B", + ), + "parent": ("..",) + } + ), + ) + + ebuilds = { + "sys-libs/A-1": { + "EAPI": "5", + }, + "app-misc/A-1": { + "EAPI": "5", + }, + "app-misc/B-1": { + "EAPI": "5", + }, + "app-misc/C-1": { + "EAPI": "5", + }, + } + + installed = { + "sys-libs/A-1": { + "EAPI": "5", + }, + "app-misc/A-1": { + "EAPI": "5", + }, + "app-misc/B-1": { + "EAPI": "5", + }, + "app-misc/C-1": { + "EAPI": "5", + }, + } + + test_cases = ( + + ResolverPlaygroundTestCase( + ["@world"], + options={"--update": True, "--deep": True}, + mergelist = [], + success = True, + ), + + ResolverPlaygroundTestCase( + [], + options={"--depclean": True}, + success=True, + cleanlist=["app-misc/B-1"] + ), + + ) + + playground = ResolverPlayground(debug=False, ebuilds=ebuilds, + installed=installed, repo_configs=repo_configs) + try: + repo_dir = (playground.settings.repositories. + get_location_for_name("test_repo")) + profile_root = os.path.join(repo_dir, "profiles") + + for p, data in profiles: + prof_path = os.path.join(profile_root, p) + ensure_dirs(prof_path) + for k, v in data.items(): + with io.open(os.path.join(prof_path, k), mode="w", + encoding=_encodings["repo.content"]) as f: + for line in v: + f.write("%s\n" % line) + + # The config must be reloaded in order to account + # for the above profile customizations. + playground.reload_config() + + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, + test_case.fail_msg) + + finally: + playground.cleanup()
