[gentoo-portage-dev] [PATCH 0/7] Add FEATURES=binpkg-multi-instance (bug 150031)
FEATURES=binpkg-multi-instance causes an integer build-id to be
associated with each binary package instance. Inclusion of the build-id
in the file name of the binary package file makes it possible to store
an arbitrary number of binary packages built from the same ebuild.
Having multiple instances is useful for a number of purposes, such as
retaining builds that were built with different USE flags or linked
against different versions of libraries. The location of any particular
package within PKGDIR can be expressed as follows:
${PKGDIR}/${CATEGORY}/${PN}/${PF}-${BUILD_ID}.xpak
The build-id starts at 1 for the first build of a particular ebuild,
and is incremented by 1 for each new build. It is possible to share a
writable PKGDIR over NFS, and locking ensures that each package added
to PKGDIR will have a unique build-id. It is not necessary to migrate
an existing PKGDIR to the new layout, since portage is capable of
working with a mixed PKGDIR layout, where packages using the old layout
are allowed to remain in place.
The new PKGDIR layout is backward-compatible with binhost clients
running older portage, since the file format is identical, the
per-package PATH attribute in the 'Packages' index directs them to
download the file from the correct URI, and they automatically use
BUILD_TIME metadata to select the latest builds.
There is currently no automated way to prune old builds from PKGDIR,
although it is possible to remove packages manually, and then run
'emaint --fix binhost' to update the ${PKGDIR}/Packages index. Support
for FEATURES=binpkg-multi-instance is planned for eclean-pkg.
X-Gentoo-Bug: 150031
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=150031
Zac Medico (7):
binpkg-multi-instance 1 of 7
binpkg-multi-instance 2 of 7
binpkg-multi-instance 3 of 7
binpkg-multi-instance 4 of 7
binpkg-multi-instance 5 of 7
binpkg-multi-instance 6 of 7
binpkg-multi-instance 7 of 7
bin/quickpkg | 1 -
man/make.conf.5| 27 +
man/portage.5 | 8 +-
pym/_emerge/Binpkg.py | 33 +-
pym/_emerge/BinpkgFetcher.py | 13 +-
pym/_emerge/BinpkgVerifier.py | 6 +-
pym/_emerge/EbuildBinpkg.py| 9 +-
pym/_emerge/EbuildBuild.py | 36 +-
pym/_emerge/Package.py | 67 +-
pym/_emerge/Scheduler.py | 6 +-
pym/_emerge/clear_caches.py| 1 -
pym/_emerge/is_valid_package_atom.py | 5 +-
pym/_emerge/resolver/output.py | 21 +-
pym/portage/_sets/ProfilePackageSet.py | 3 +-
pym/portage/_sets/profiles.py | 3 +-
pym/portage/const.py | 2 +
pym/portage/dbapi/__init__.py | 10 +-
pym/portage/dbapi/bintree.py | 843 +++--
pym/portage/dbapi/vartree.py | 8 +-
pym/portage/dbapi/virtual.py | 113 ++-
pym/portage/dep/__init__.py| 35 +-
pym/portage/emaint/modules/binhost/binhost.py | 47 +-
.../package/ebuild/_config/KeywordsManager.py | 3 +-
.../package/ebuild/_config/LocationsManager.py | 8 +-
pym/portage/package/ebuild/_config/MaskManager.py | 21 +-
pym/portage/package/ebuild/_config/UseManager.py | 14 +-
pym/portage/package/ebuild/config.py | 15 +-
pym/portage/repository/config.py | 2 +-
pym/portage/tests/dep/test_isvalidatom.py | 8 +-
pym/portage/tests/resolver/ResolverPlayground.py | 25 +-
.../resolver/binpkg_multi_instance/__init__.py | 2 +
.../resolver/binpkg_multi_instance/__test__.py | 2 +
.../test_build_id_profile_format.py| 134
.../binpkg_multi_instance/test_rebuilt_binaries.py | 101 +++
pym/portage/util/__init__.py | 13 +-
pym/portage/versions.py| 28 +-
36 files changed, 1144 insertions(+), 529 deletions(-)
create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/__init__.py
create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/__test__.py
create mode 100644
pym/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py
create mode 100644
pym/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
--
2.0.5
[gentoo-portage-dev] [PATCH 1/7] binpkg-multi-instance 1 of 7
Extend the _pkg_str class with build_id, build_time, file_size, and
mtime attributes. These will be used to distinguish binary package
instances that have the same cpv. Package sorting accounts for
build_time, which will be used to prefer newer builds over older builds
when their versions are identical.
---
pym/_emerge/Package.py | 51 +-
pym/_emerge/resolver/output.py | 21 +
pym/portage/dbapi/__init__.py | 10 +++--
pym/portage/dbapi/vartree.py | 8 +--
pym/portage/versions.py| 28 +--
5 files changed, 92 insertions(+), 26 deletions(-)
diff --git a/pym/_emerge/Package.py b/pym/_emerge/Package.py
index e8a13cb..975335d 100644
--- a/pym/_emerge/Package.py
+++ b/pym/_emerge/Package.py
@@ -41,12 +41,12 @@ class Package(Task):
_validated_atoms, _visible)
metadata_keys = [
- BUILD_TIME, CHOST, COUNTER, DEPEND, EAPI,
- HDEPEND, INHERITED, IUSE, KEYWORDS,
- LICENSE, PDEPEND, PROVIDE, RDEPEND,
- repository, PROPERTIES, RESTRICT, SLOT, USE,
- _mtime_, DEFINED_PHASES, REQUIRED_USE, PROVIDES,
- REQUIRES]
+ BUILD_ID, BUILD_TIME, CHOST, COUNTER, DEFINED_PHASES,
+ DEPEND, EAPI, HDEPEND, INHERITED, IUSE, KEYWORDS,
+ LICENSE, MD5, PDEPEND, PROVIDE, PROVIDES,
+ RDEPEND, repository, REQUIRED_USE,
+ PROPERTIES, REQUIRES, RESTRICT, SIZE,
+ SLOT, USE, _mtime_]
_dep_keys = ('DEPEND', 'HDEPEND', 'PDEPEND', 'RDEPEND')
_buildtime_keys = ('DEPEND', 'HDEPEND')
@@ -114,13 +114,14 @@ class Package(Task):
return self._metadata[EAPI]
@property
+ def build_id(self):
+ return self.cpv.build_id
+
+ @property
def build_time(self):
if not self.built:
raise AttributeError('build_time')
- try:
- return long(self._metadata['BUILD_TIME'])
- except (KeyError, ValueError):
- return 0
+ return self.cpv.build_time
@property
def defined_phases(self):
@@ -509,9 +510,15 @@ class Package(Task):
else:
cpv_color = PKG_NOMERGE
+ build_id_str =
+ if isinstance(self.cpv.build_id, long) and self.cpv.build_id
0:
+ build_id_str = -%s % self.cpv.build_id
+
s = (%s, %s \
- % (portage.output.colorize(cpv_color, self.cpv +
_slot_separator + \
- self.slot + / + self.sub_slot + _repo_separator +
self.repo) , self.type_name)
+ % (portage.output.colorize(cpv_color, self.cpv +
+ build_id_str + _slot_separator + self.slot + / +
+ self.sub_slot + _repo_separator + self.repo),
+ self.type_name)
if self.type_name == installed:
if self.root_config.settings['ROOT'] != /:
@@ -755,29 +762,41 @@ class Package(Task):
def __lt__(self, other):
if other.cp != self.cp:
return self.cp other.cp
- if portage.vercmp(self.version, other.version) 0:
+ result = portage.vercmp(self.version, other.version)
+ if result 0:
return True
+ if result == 0 and self.built and other.built:
+ return self.build_time other.build_time
return False
def __le__(self, other):
if other.cp != self.cp:
return self.cp = other.cp
- if portage.vercmp(self.version, other.version) = 0:
+ result = portage.vercmp(self.version, other.version)
+ if result = 0:
return True
+ if result == 0 and self.built and other.built:
+ return self.build_time = other.build_time
return False
def __gt__(self, other):
if other.cp != self.cp:
return self.cp other.cp
- if portage.vercmp(self.version, other.version) 0:
+ result = portage.vercmp(self.version, other.version)
+ if result 0:
return True
+ if result == 0 and self.built and other.built:
+ return self.build_time other.build_time
return False
def __ge__(self, other):
if other.cp != self.cp:
return self.cp = other.cp
- if portage.vercmp(self.version, other.version) = 0:
+ result = portage.vercmp(self.version, other.version)
+ if result = 0:
return True
+ if
[gentoo-portage-dev] [PATCH 3/7] binpkg-multi-instance 3 of 7
FEATURES=binpkg-multi-instance causes an integer build-id to be
associated with each binary package instance. Inclusion of the build-id
in the file name of the binary package file makes it possible to store
an arbitrary number of binary packages built from the same ebuild.
Having multiple instances is useful for a number of purposes, such as
retaining builds that were built with different USE flags or linked
against different versions of libraries. The location of any particular
package within PKGDIR can be expressed as follows:
${PKGDIR}/${CATEGORY}/${PN}/${PF}-${BUILD_ID}.xpak
The build-id starts at 1 for the first build of a particular ebuild,
and is incremented by 1 for each new build. It is possible to share a
writable PKGDIR over NFS, and locking ensures that each package added
to PKGDIR will have a unique build-id. It is not necessary to migrate
an existing PKGDIR to the new layout, since portage is capable of
working with a mixed PKGDIR layout, where packages using the old layout
are allowed to remain in place.
The new PKGDIR layout is backward-compatible with binhost clients
running older portage, since the file format is identical, the
per-package PATH attribute in the 'Packages' index directs them to
download the file from the correct URI, and they automatically use
BUILD_TIME metadata to select the latest builds.
There is currently no automated way to prune old builds from PKGDIR,
although it is possible to remove packages manually, and then run
'emaint --fix binhost' to update the ${PKGDIR}/Packages index. Support
for FEATURES=binpkg-multi-instance is planned for eclean-pkg.
X-Gentoo-Bug: 150031
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=150031
---
bin/quickpkg | 1 -
man/make.conf.5 | 27 +
pym/_emerge/Binpkg.py | 33 +-
pym/_emerge/BinpkgFetcher.py | 13 +-
pym/_emerge/BinpkgVerifier.py | 6 +-
pym/_emerge/EbuildBinpkg.py | 9 +-
pym/_emerge/EbuildBuild.py| 36 +-
pym/_emerge/Package.py| 16 +-
pym/_emerge/Scheduler.py | 6 +-
pym/_emerge/clear_caches.py | 1 -
pym/portage/const.py | 2 +
pym/portage/dbapi/bintree.py | 684 +-
pym/portage/emaint/modules/binhost/binhost.py | 47 +-
13 files changed, 601 insertions(+), 280 deletions(-)
diff --git a/bin/quickpkg b/bin/quickpkg
index 2c69a69..8b71c3e 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -63,7 +63,6 @@ def quickpkg_atom(options, infos, arg, eout):
pkgs_for_arg = 0
for cpv in matches:
excluded_config_files = []
- bintree.prevent_collision(cpv)
dblnk = vardb._dblink(cpv)
have_lock = False
diff --git a/man/make.conf.5 b/man/make.conf.5
index 84b7191..6ead61b 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -256,6 +256,33 @@ has a \fB\-\-force\fR option that can be used to force
regeneration of digests.
Keep logs from successful binary package merges. This is relevant only when
\fBPORT_LOGDIR\fR is set.
.TP
+.B binpkg\-multi\-instance
+Enable support for multiple binary package instances per ebuild.
+Having multiple instances is useful for a number of purposes, such as
+retaining builds that were built with different USE flags or linked
+against different versions of libraries. The location of any particular
+package within PKGDIR can be expressed as follows:
+
+ ${PKGDIR}/${CATEGORY}/${PN}/${PF}\-${BUILD_ID}.xpak
+
+The build\-id starts at 1 for the first build of a particular ebuild,
+and is incremented by 1 for each new build. It is possible to share a
+writable PKGDIR over NFS, and locking ensures that each package added
+to PKGDIR will have a unique build\-id. It is not necessary to migrate
+an existing PKGDIR to the new layout, since portage is capable of
+working with a mixed PKGDIR layout, where packages using the old layout
+are allowed to remain in place.
+
+The new PKGDIR layout is backward\-compatible with binhost clients
+running older portage, since the file format is identical, the
+per\-package PATH attribute in the 'Packages' index directs them to
+download the file from the correct URI, and they automatically use
+BUILD_TIME metadata to select the latest builds.
+
+There is currently no automated way to prune old builds from PKGDIR,
+although it is possible to remove packages manually, and then run
+\(aqemaint \-\-fix binhost' to update the ${PKGDIR}/Packages index.
+.TP
.B buildpkg
Binary packages will be created for all packages that are merged. Also see
\fBquickpkg\fR(1) and \fBemerge\fR(1) \fB\-\-buildpkg\fR and
diff --git a/pym/_emerge/Binpkg.py b/pym/_emerge/Binpkg.py
index ded6dfd..7b7ae17 100644
--- a/pym/_emerge/Binpkg.py
+++ b/pym/_emerge/Binpkg.py
@@ -121,16 +121,11 @@ class
[gentoo-portage-dev] [PATCH 7/7] binpkg-multi-instance 7 of 7
Support profile-formats = build-id setting for layout.conf. When this is enabled in layout.conf of the containing repository, a dependency atom in the profile can refer to a specific build, using the build-id that is assigned when FEATURES=binpkg-multi-instance is enabled. A build-id atom is identical to a version-specific atom, except that the version is followed by a hyphen and an integer build-id. With the build-id profile format, it is possible to assemble a system using specific builds of binary packages, as users of binary distros might be accustomed to. For example, an atom in the packages file can pull a specific build of a package into the @system set, and an atom in the package.keywords file can be used to modify the effective KEYWORDS of a specific build of a package. Refering to specific builds can be useful for a number of reasons. For example, if a particular build needs to undergo a large amount of testing in a complex environment in order to verify reliability, then it can be useful to lock a profile to a specific build that has been thoroughly tested. --- This patch is identical to [PATCH 2/2] Add profile-formats=build-id (bug 150031) which was sent to the list earlier, except that it adds a comment about smart defaults in the Atom class. man/portage.5 | 8 +- pym/_emerge/is_valid_package_atom.py | 5 +- pym/portage/_sets/ProfilePackageSet.py | 3 +- pym/portage/_sets/profiles.py | 3 +- pym/portage/dep/__init__.py| 38 +- .../package/ebuild/_config/KeywordsManager.py | 3 +- .../package/ebuild/_config/LocationsManager.py | 8 +- pym/portage/package/ebuild/_config/MaskManager.py | 21 +++- pym/portage/package/ebuild/_config/UseManager.py | 14 ++- pym/portage/package/ebuild/config.py | 15 ++- pym/portage/repository/config.py | 2 +- pym/portage/tests/dep/test_isvalidatom.py | 8 +- .../test_build_id_profile_format.py| 134 + pym/portage/util/__init__.py | 13 +- 14 files changed, 237 insertions(+), 38 deletions(-) create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py diff --git a/man/portage.5 b/man/portage.5 index ed5140d..e062f9f 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -1180,7 +1180,7 @@ and the newer/faster md5-dict format. Default is to detect dirs. The EAPI to use for profiles when unspecified. This attribute is supported only if profile-default-eapi is included in profile-formats. .TP -.BR profile\-formats = [pms] [portage-1] [portage-2] [profile-bashrcs] [profile-set] [profile-default-eapi] +.BR profile\-formats = [pms] [portage-1] [portage-2] [profile-bashrcs] [profile-set] [profile-default-eapi] [build-id] Control functionality available to profiles in this repo such as which files may be dirs, or the syntax available in parent files. Use portage-2 if you're unsure. The default is portage-1-compat mode which is meant to be compatible @@ -1190,7 +1190,11 @@ Setting profile-bashrcs will enable the per-profile bashrc mechanism profile \fBpackages\fR file to add atoms to the @profile package set. See the profile \fBpackages\fR section for more information. Setting profile-default-eapi enables support for the -profile_eapi_when_unspecified attribute. +profile_eapi_when_unspecified attribute. Setting build\-id allows +dependency atoms in the profile to refer to specific builds (see the +binpkg\-multi\-instance FEATURES setting in \fBmake.conf\fR(5)). A +build\-id atom is identical to a version-specific atom, except that the +version is followed by a hyphen and an integer build\-id. .RE .RE diff --git a/pym/_emerge/is_valid_package_atom.py b/pym/_emerge/is_valid_package_atom.py index 112afc1..17f7642 100644 --- a/pym/_emerge/is_valid_package_atom.py +++ b/pym/_emerge/is_valid_package_atom.py @@ -14,9 +14,10 @@ def insert_category_into_atom(atom, category): ret = None return ret -def is_valid_package_atom(x, allow_repo=False): +def is_valid_package_atom(x, allow_repo=False, allow_build_id=True): if / not in x.split(:)[0]: x2 = insert_category_into_atom(x, 'cat') if x2 != None: x = x2 - return isvalidatom(x, allow_blockers=False, allow_repo=allow_repo) + return isvalidatom(x, allow_blockers=False, allow_repo=allow_repo, + allow_build_id=allow_build_id) diff --git a/pym/portage/_sets/ProfilePackageSet.py b/pym/portage/_sets/ProfilePackageSet.py index 2fcafb6..fec9373 100644 --- a/pym/portage/_sets/ProfilePackageSet.py +++ b/pym/portage/_sets/ProfilePackageSet.py @@ -23,7 +23,8 @@ class ProfilePackageSet(PackageSet): def load(self): self._setAtoms(x for x in stack_lists(
[gentoo-portage-dev] [PATCH 4/7] binpkg-multi-instance 4 of 7
Add a test case to verify that emerge --rebuilt-binaries works with
binpkg-multi-instance. This relies on the fact that binary packages of
the same version are ordered by BUILD_TIME, so that the latest builds
are preferred when appropriate.
---
pym/portage/tests/resolver/ResolverPlayground.py | 25 -
.../resolver/binpkg_multi_instance/__init__.py | 2 +
.../resolver/binpkg_multi_instance/__test__.py | 2 +
.../binpkg_multi_instance/test_rebuilt_binaries.py | 101 +
4 files changed, 126 insertions(+), 4 deletions(-)
create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/__init__.py
create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/__test__.py
create mode 100644
pym/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
diff --git a/pym/portage/tests/resolver/ResolverPlayground.py
b/pym/portage/tests/resolver/ResolverPlayground.py
index 84ad17c..6bdf2c7 100644
--- a/pym/portage/tests/resolver/ResolverPlayground.py
+++ b/pym/portage/tests/resolver/ResolverPlayground.py
@@ -39,6 +39,7 @@ class ResolverPlayground(object):
config_files = frozenset((eapi, layout.conf, make.conf,
package.accept_keywords,
package.keywords, package.license, package.mask,
package.properties,
+ package.provided, packages,
package.unmask, package.use, package.use.aliases,
package.use.stable.mask,
soname.provided,
unpack_dependencies, use.aliases, use.force, use.mask,
layout.conf))
@@ -208,12 +209,18 @@ class ResolverPlayground(object):
raise AssertionError('digest creation failed
for %s' % ebuild_path)
def _create_binpkgs(self, binpkgs):
- for cpv, metadata in binpkgs.items():
+ # When using BUILD_ID, there can be mutiple instances for the
+ # same cpv. Therefore, binpkgs may be an iterable instead of
+ # a dict.
+ items = getattr(binpkgs, 'items', None)
+ items = items() if items is not None else binpkgs
+ for cpv, metadata in items:
a = Atom(= + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = test_repo
+ pn = catsplit(a.cp)[1]
cat, pf = catsplit(a.cpv)
metadata = metadata.copy()
metadata.setdefault(SLOT, 0)
@@ -225,8 +232,13 @@ class ResolverPlayground(object):
repo_dir = self.pkgdir
category_dir = os.path.join(repo_dir, cat)
- binpkg_path = os.path.join(category_dir, pf + .tbz2)
- ensure_dirs(category_dir)
+ if BUILD_ID in metadata:
+ binpkg_path = os.path.join(category_dir, pn,
+ %s-%s.xpak% (pf,
metadata[BUILD_ID]))
+ else:
+ binpkg_path = os.path.join(category_dir, pf +
.tbz2)
+
+ ensure_dirs(os.path.dirname(binpkg_path))
t = portage.xpak.tbz2(binpkg_path)
t.recompose_mem(portage.xpak.xpak_mem(metadata))
@@ -252,6 +264,7 @@ class ResolverPlayground(object):
unknown_keys = set(metadata).difference(
portage.dbapi.dbapi._known_keys)
unknown_keys.discard(BUILD_TIME)
+ unknown_keys.discard(BUILD_ID)
unknown_keys.discard(COUNTER)
unknown_keys.discard(repository)
unknown_keys.discard(USE)
@@ -749,7 +762,11 @@ class ResolverPlaygroundResult(object):
repo_str =
if x.repo != test_repo:
repo_str = _repo_separator +
x.repo
- mergelist_str = x.cpv + repo_str
+ build_id_str =
+ if (x.type_name == binary and
+ x.cpv.build_id is not None):
+ build_id_str = -%s %
x.cpv.build_id
+ mergelist_str = x.cpv + build_id_str +
repo_str
if x.built:
if x.operation == merge:
desc = x.type_name
diff --git a/pym/portage/tests/resolver/binpkg_multi_instance/__init__.py
b/pym/portage/tests/resolver/binpkg_multi_instance/__init__.py
new file mode 100644
index 000..4725d33
--- /dev/null
+++
[gentoo-portage-dev] Re: [PATCH 1/2] Add FEATURES=binpkg-multi-instance (bug 150031)
Duncan posted on Wed, 18 Feb 2015 03:40:35 + as excerpted: Reading this gave me a distinct sense of deja vu reading this. ... =:^P Crossed in the mail. Already corrected in the new series. =:^) -- Duncan - List replies preferred. No HTML msgs. Every nonfree program has a lord, a master -- and if you use the program, he is your master. Richard Stallman
[gentoo-dev] Re: [gentoo-commits] gentoo-x86 commit in dev-libs/libusbhp: ChangeLog Manifest libusbhp-1.0.2.ebuild metadata.xml
Joshua Kinard posted on Tue, 17 Feb 2015 12:46:12 -0500 as excerpted: On 02/16/2015 13:01, Rich Freeman wrote: On Mon, Feb 16, 2015 at 11:02 AM, Joshua Kinard ku...@gentoo.org wrote: On 02/16/2015 09:04, Rich Freeman wrote: Maybe another approach is to just ditch per-file copyrights entirely (which a random perusal suggests is how Linux does things), but that would STILL require stripping the copyright out of these files with all the issues that entails, and limit our ability to borrow license-compatible code. [M]y understanding is the kernel retains per-file copyrights. This is why the kernel is permanently wedded to GPLv2, because some of the contributors owning those copyrights have died and thus can no longer consent to changing to the GPLv3[.] Perhaps I should have worded that better. s/per-file copyrights/per-file copyright notices/ Obviously the content of individual files will always be copyrighted absent a release into the public domain. The Linux kernel just doesn't stick notices on individual files that attempt to identify who owns the copyright on what. Presumably that also means that if they borrow a file from somewhere else they don't care to change the copyright notice that was already there (or somehow they manage to avoid the euthusiasm stirred up by removing said notices). Well, I just sent a patch upstream that adds a new RTC driver to the kernel, and I added copyright to myself and the guy that created the original driver that I based off of to the top of the source file (and its header). So that practice is still used, and akpm recently added it to -mm with no comment on any of the copyright bits, so I must've gotten part that right. It's probably left to the person writing the specific source file(s) on how they want to do copyright, as long as they stick to recognized norms and GPLv2. The kernel's relatively relaxed per-file copyright and license policy is in the context of git and its record of a rather strong explicit per-commit signed-off-by policy. As a result of the strong per-commit signed-off-by policy, they can be and are relatively more relaxed on a per-file policy, since the sign-off policy requires legal responsibility and the authority to grant default-gpl2-only permissions on anything committed in the first place. As such, any file without an explicit license CAN BE ASSUMED to have the GPLv2 license, and copyright CAN BE ASSUMED to remain with the original author (company in the case of a work- for-hire unless otherwise stated), because that's part of the conditions that are agreed to by the explicit signed-off-by. Since gentoo lacks this sort of formal signed-off policy and in fact has yet to move to git where it could be most easily tracked and enforced (let alone such a policy created and formally agreed in the first place), the extent to which the kernel's relatively relaxed per-file policies could apply to gentoo in its current cvs and policy state is rather limited. IOW, the kernel's policy doesn't apply here, except to the extent that we use it as a goal/model to increase the urgency of the switch to git, and once having done so, creating and adopting a similarly strict per-commit- sign-off basic policy context in which to apply a similarly relaxed per- file policy. -- Duncan - List replies preferred. No HTML msgs. Every nonfree program has a lord, a master -- and if you use the program, he is your master. Richard Stallman
[gentoo-portage-dev] [PATCH 6/7] binpkg-multi-instance 6 of 7
Remove unused binarytree _remove_symlink, _create_symlink,
prevent_collision, _move_to_all, and _move_from_all methods. These are
all related to the oldest PKGDIR layout, which put all of the tbz2
files in $PKGDIR/All, and created symlinks to them in the category
directories. The $PKGDIR/All layout should be practically extinct by
now. Now portage recognizes all existing layouts, or mixtures of them,
and uses the old packages in place. It never puts new packages in
$PKGDIR/All, so there's no need to move packages around to prevent file
name collisions between packages from different categories. It also
only uses regular files (any symlinks are ignored).
---
pym/portage/dbapi/bintree.py | 117 ++-
1 file changed, 4 insertions(+), 113 deletions(-)
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 5636a5f..a475fb5 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -471,89 +471,11 @@ class binarytree(object):
return moves
- def _remove_symlink(self, cpv):
- Remove a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink and also
remove
- the ${PKGDIR}/${CATEGORY} directory if empty. The file will
not be
- removed if os.path.islink() returns False.
- mycat, mypkg = catsplit(cpv)
- mylink = os.path.join(self.pkgdir, mycat, mypkg + .tbz2)
- if os.path.islink(mylink):
- Only remove it if it's really a link so that this
method never
- removes a real package that was placed here to avoid a
collision.
- os.unlink(mylink)
- try:
- os.rmdir(os.path.join(self.pkgdir, mycat))
- except OSError as e:
- if e.errno not in (errno.ENOENT,
- errno.ENOTEMPTY, errno.EEXIST):
- raise
- del e
-
- def _create_symlink(self, cpv):
- Create a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink (and
- ${PKGDIR}/${CATEGORY} directory, if necessary). Any file that
may
- exist in the location of the symlink will first be removed.
- mycat, mypkg = catsplit(cpv)
- full_path = os.path.join(self.pkgdir, mycat, mypkg + .tbz2)
- self._ensure_dir(os.path.dirname(full_path))
- try:
- os.unlink(full_path)
- except OSError as e:
- if e.errno != errno.ENOENT:
- raise
- del e
- os.symlink(os.path.join(.., All, mypkg + .tbz2),
full_path)
-
def prevent_collision(self, cpv):
- Make sure that the file location ${PKGDIR}/All/${PF}.tbz2 is
safe to
- use for a given cpv. If a collision will occur with an existing
- package from another category, the existing package will be
bumped to
- ${PKGDIR}/${CATEGORY}/${PF}.tbz2 so that both can coexist.
- if not self._all_directory:
- return
-
- # Copy group permissions for new directories that
- # may have been created.
- for path in (All, catsplit(cpv)[0]):
- path = os.path.join(self.pkgdir, path)
- self._ensure_dir(path)
- if not os.access(path, os.W_OK):
- raise PermissionDenied(access('%s', W_OK) %
path)
-
- full_path = self.getname(cpv)
- if All == full_path.split(os.path.sep)[-2]:
- return
- Move a colliding package if it exists. Code below this
point only
- executes in rare cases.
- mycat, mypkg = catsplit(cpv)
- myfile = mypkg + .tbz2
- mypath = os.path.join(All, myfile)
- dest_path = os.path.join(self.pkgdir, mypath)
-
- try:
- st = os.lstat(dest_path)
- except OSError:
- st = None
- else:
- if stat.S_ISLNK(st.st_mode):
- st = None
- try:
- os.unlink(dest_path)
- except OSError:
- if os.path.exists(dest_path):
- raise
-
- if st is not None:
- # For invalid packages, other_cat could be None.
- other_cat =
portage.xpak.tbz2(dest_path).getfile(bCATEGORY)
- if other_cat:
- other_cat = _unicode_decode(other_cat,
- encoding=_encodings['repo.content'],
[gentoo-dev] Re: About reducing or even removing stable tree for some arches
Pacho Ramos posted on Mon, 16 Feb 2015 14:34:50 +0100 as excerpted: The current policy of maintainers dropping keywords after 90 days is simply not applied because it leads up to that maintainer needing to kill himself that keyword and ALL the reverse deps keywords and, then, all that effort should probably be replaced by making the opposite, I mean, reducing the stable tree of that arches to a minimum and moving all the other packages to testing. The main advantage of this is that it needs maybe more effort in one round but it solves the problem for the future. On the other hand trying to kill keywords of a package *and all its reverse deps* requires a lot of work every time the problem appears. Perhaps my non-dev status prevents me from understanding the difficulty here, but... I really don't see the problem. 1) I don't believe the 90-day policy was /supposed/ to be particularly easy. It was supposed to be a pressure relief valve, to release pressure only if it built up beyond a certain level, such that both archs and package-devs could still live with the situation by keeping the pressure from going off the scale at either location. As a pressure reliever, what you defined as a bug I'd rather define as a feature. If the situation gets bad enough and the pressure high enough, there's an approved method to relieve it, but that method itself isn't entirely pain-free, so it doesn't tend to be used until the pain of not using it is worse than the pain of using it, which, I'd argue, is functioning as intended. 2) The very requirement of having to kill ALL the reverse-deps seems to me to already lessen the pressure necessary to tip the balance the next time, since either it's not the problem its made out to be if it's only a handful of packages, or within a few cycles of doing this, there will be dramatically fewer packages keyworded in the first place to worry about, and thus dramatically fewer packages to have to dekeyword this time around. Yes, the first time's going to be hell. And the second time could easily be 90-95% as bad, particularly if the two packages are in separate areas and there's little overlap. But the tenth? By then, the number of packages still keyworded in the first place should be down enough to make a difference. And the 20th? By then, things should be much more reasonable. So you're suggesting a flag day and volunteering to do most of the work. I won't argue with that as I don't have a dog in this fight. But it seems to me, by the time you do even say five existing 90-day- dekeywording-policy actions, you'll either have something already looking a lot more like the goal you outline above than the current state and will be well on your way, or if that DIDN'T dekeyword enough packages to already be easier, then by definition there's only a handful of such dependencies in the first place. Either way, I simply don't see the problem, certainly so when comparing the work of just doing it under the existing policy, to fighting the political war necessary to change it -- and even assuming a win, ending up dekeywording pretty much the same set of packages as you'd have done with a few rounds under the existing policy anyway. Again, not that I disagree. As I said, no dog in this fight and I might actually benefit by the developer time then freed up to work on fights I do have a dog in. But I expect this question will come up in some form in any case, and by answering it now, it'll already be dealt with. Plus, I'm simply curious, as there's evidently an angle I'm blind to, and now being aware of that blindness, it's disturbing enough to me that I want to be rid of it, thus the question. =:^) -- Duncan - List replies preferred. No HTML msgs. Every nonfree program has a lord, a master -- and if you use the program, he is your master. Richard Stallman
Re: [gentoo-dev] Re: [gentoo-commits] gentoo-x86 commit in dev-libs/libusbhp: ChangeLog Manifest libusbhp-1.0.2.ebuild metadata.xml
On Tue, Feb 17, 2015 at 9:19 PM, Duncan 1i5t5.dun...@cox.net wrote: Since gentoo lacks this sort of formal signed-off policy and in fact has yet to move to git where it could be most easily tracked and enforced (let alone such a policy created and formally agreed in the first place), the extent to which the kernel's relatively relaxed per-file policies could apply to gentoo in its current cvs and policy state is rather limited. IOW, the kernel's policy doesn't apply here, except to the extent that we use it as a goal/model to increase the urgency of the switch to git, and once having done so, creating and adopting a similarly strict per-commit- sign-off basic policy context in which to apply a similarly relaxed per- file policy. I was thinking more for after the git migration and we have a DCO. A big part of what was holding me back from pushing more on the new policy is the fact that the bookkeeping looks potentially onerous. If we could simplify things and be compliant and just have a simple DCO and optional FLA, then there isn't a lot holding us back besides git (and maybe we can find a way around that if we're desperate). -- Rich
[gentoo-dev] ebuild copyright assignment
On Mon, 16 Feb 2015 06:39:51 -0500 Mike Frysinger vap...@gentoo.org wrote: the policy is not it must be Gentoo copyright, but it must have a header that says Gentoo copyright even though there's no legal basis for it. Correct, but I have my doubts about the allegedly wobbly legal basis. I do vividly recall reading these: http://www.gentoo.org/proj/en/devrel/copyright/index.xml http://web.archive.org/web/20040604022011/http://www.gentoo.org/doc/en/policy.xml Copyright in ebuilds (and documentation) should always be assigned to Gentoo Technologies. Developers must never put their own names in copyright lines. For more information, please see http://www.gentoo.org/proj/en/devrel/copyright-assignment.xml http://web.archive.org/web/20040624223240/http://www.gentoo.org/proj/en/devrel/copyright-assignment.xml (Page moved to http://www.gentoo.org/proj/en/devrel/copyright/index.xml http://web.archive.org/web/20040618235041/http://www.gentoo.org/proj/en/devrel/copyright/index.xml) This was the pseudo-legal language in place when I became a developer, and as of this day I still assume all ebuilds' copyright MUST be assigned to the project. The language does seem to have disappeared from the website, though. Regardless, the mechanism was that by way of adding that header, you assign all rights to the Gentoo Foundation, nee Gentoo Technologies. I seem to recall the developer quizzes may have had (or indeed requested) some more information on this matter. I seem to recall the wobbly legal basis assumed that the entire ebuild format was copyrighted, which I would agree is unenforceable. But the language that used to say all ebuilds' copyrights should be assigned to [Gentoo] would still hold. Note that I am not talking about QA actions or other trivial stuff that happened in the tree one day here - I'm just wondering where the legal language from 2004 went. jer
Re: [gentoo-dev] ebuild copyright assignment
On 18/02/15 08:40, Jeroen Roovers wrote: I seem to recall the developer quizzes may have had (or indeed requested) some more information on this matter. The test ebuild focuses on this topic. signature.asc Description: OpenPGP digital signature
Re: [gentoo-dev] [PATCH] cmake-utils.eclass: add Fortran to Gentoo override rules (set valid compiler, append FCFLAGS)
On Wed, 11 Feb 2015 11:56:37 -0700 Christoph Junghans wrote: 2015-02-11 11:14 GMT-07:00 Andrew Savchenko birc...@gentoo.org: Hello, attached patch adds Fortran compiler to Gentoo override rules in cmake-utils.eclass the same way C/C++ compilers are added. This change is needed because packages which force their own precedence of Fortran compilers or flags also exists. We hit this issue on bug 486626 [1] for sci-libs/lapacke-reference (science overlay): if multiple Fortran compilers are installed, e.g. pathf95 and gfortran, package prefers pathf95 — this may even broke build, because most FCFLAGS are not compatible between this compilers In order to fix this FC should be set from $(tc-getFC), which is implemented in proposed patch as well as ensuring proper flags are set. This change was already discussed with KDE team [1] and I'm going to commit this patch after feedback or in a week if there are no objections. +1 Ok. Fix is in tree now. Best regards, Andrew Savchenko pgpLKVaYU_YoE.pgp Description: PGP signature
Re: [gentoo-portage-dev] [PATCH 1/2] Add FEATURES=binpkg-multi-instance (bug 150031)
On 02/17/2015 10:42 AM, Brian Dolbec wrote: overall, there is no way I know the code well enough to know if you screwed up. But the code looks decent, so... My only questions are: pym/portage/dbapi/bintree.py: You removed several functions from the binarytree class and essentially reduced prevent_collision to a warning message. Can you briefly say why they are not needed please. Okay, I'll include this info in an updated patch: _pkgindex_cpv_map_latest_build: This is what binhost clients running older versions of portage will use to select to select the latest builds when their binhost server switches to FEATURES=binpkg-multi-instance. New portage won't need this anymore because it is capable of examining multiple builds and it uses sorting to ensure that the latest builds are preferred when appropriate. _remove_symlink, _create_symlink, prevent_collision, _move_to_all, and _move_from_all: These are all related to the oldest PKGDIR layout, which put all of the tbz2 files in $PKGDIR/All, and created symlinks to them in the category directories. The $PKGDIR/All layout should be practically extinct by now. New portage recognizes all existing layouts, or mixtures of them, and uses the old packages in place. It never puts new packages in $PKGDIR/All, so there's no need to move packages around to prevent file name collisions between packages from different categories. It also only uses regular files (any symlinks are ignored). _populate() == it is some 500+ LOC nasty. From what I can see I think you added slightly more loc than you deleted/changed. While I don't expect a breakup of this function to be directly a part of this commit. IT IS BADLY NEEDED. I'd like to see this function properly split into pieces and act as a driver for the smaller tasks. Currently it is a mess of nested if/else, for,... that is extremely difficult to keep straight and make logic changes to. It's almost as bad as repomans 1000+ LOC main loop. Why if TRUE: == just fix the indent... At the same time, I noticed _ensure_dirs, _file_permissions functions defined in that class. Shouldn't these just be util functions available everywhere. Yeah, I will work on cleaning that up, and submit it as a separate patch. pym/portage/versions.py class _pkg_str(_unicode) in __init__() if build_time is not None: try: build_time = long(build_time) except ValueError: if build_time: build_time = -1 else: build_time = 0 if build_id is not None: try: build_id = long(build_id) except ValueError: if build_id: build_id = -1 else: build_id = None if file_size is not None: try: file_size = long(file_size) except ValueError: if file_size: file_size = -1 else: file_size = None if mtime is not None: try: mtime = long(mtime) except ValueError: if mtime: mtime = -1 else: mtime = None I hate repeating code. This can be done using one universal small function and will clean up the __init__() @static_method def _long(var, default): if var is not None: try: var = long(var) except ValueError: if var: var = -1 else: var = default return var Okay, I'll do that and post the updated patch. -- Thanks, Zac
Re: [gentoo-portage-dev] [PATCH 1/2] Add FEATURES=binpkg-multi-instance (bug 150031)
On Tue, 17 Feb 2015 11:26:27 -0800 Zac Medico zmed...@gentoo.org wrote: On 02/17/2015 10:42 AM, Brian Dolbec wrote: overall, there is no way I know the code well enough to know if you screwed up. But the code looks decent, so... My only questions are: pym/portage/dbapi/bintree.py: You removed several functions from the binarytree class and essentially reduced prevent_collision to a warning message. Can you briefly say why they are not needed please. Okay, I'll include this info in an updated patch: Actually, I think this one patch could be split into a few logical ones. Tag them binpkg-multi-instance 1 of... in the commit message so it is clear they belong together. That way the commit messages can more clearly be relevant to the file(s) changed. The commit message is already a short story in length ;) before adding these new explanations. _pkgindex_cpv_map_latest_build: This is what binhost clients running older versions of portage will use to select to select the latest builds when their binhost server switches to FEATURES=binpkg-multi-instance. New portage won't need this anymore because it is capable of examining multiple builds and it uses sorting to ensure that the latest builds are preferred when appropriate. _remove_symlink, _create_symlink, prevent_collision, _move_to_all, and _move_from_all: These are all related to the oldest PKGDIR layout, which put all of the tbz2 files in $PKGDIR/All, and created symlinks to them in the category directories. The $PKGDIR/All layout should be practically extinct by now. New portage recognizes all existing layouts, or mixtures of them, and uses the old packages in place. It never puts new packages in $PKGDIR/All, so there's no need to move packages around to prevent file name collisions between packages from different categories. It also only uses regular files (any symlinks are ignored). _populate() == it is some 500+ LOC nasty. From what I can see I think you added slightly more loc than you deleted/changed. While I don't expect a breakup of this function to be directly a part of this commit. IT IS BADLY NEEDED. I'd like to see this function properly split into pieces and act as a driver for the smaller tasks. Currently it is a mess of nested if/else, for,... that is extremely difficult to keep straight and make logic changes to. It's almost as bad as repomans 1000+ LOC main loop. Why if TRUE: == just fix the indent... At the same time, I noticed _ensure_dirs, _file_permissions functions defined in that class. Shouldn't these just be util functions available everywhere. Yeah, I will work on cleaning that up, and submit it as a separate patch. pym/portage/versions.py class _pkg_str(_unicode) in __init__() if build_time is not None: try: build_time = long(build_time) except ValueError: if build_time: build_time = -1 else: build_time = 0 if build_id is not None: try: build_id = long(build_id) except ValueError: if build_id: build_id = -1 else: build_id = None if file_size is not None: try: file_size = long(file_size) except ValueError: if file_size: file_size = -1 else: file_size = None if mtime is not None: try: mtime = long(mtime) except ValueError: if mtime: mtime = -1 else: mtime = None I hate repeating code. This can be done using one universal small function and will clean up the __init__() @static_method def _long(var, default): if var is not None: try: var = long(var) except ValueError: if var: var = -1 else: var = default return var Okay, I'll do that and post the updated patch. -- Brian Dolbec dolsen
Re: [gentoo-portage-dev] [PATCH] binarytree: avoid unecessary index regeneration
-BEGIN PGP SIGNED MESSAGE- Hash: SHA256 On 17/02/15 18:34, Brian Dolbec wrote: SURE! just reply to the EASY one ;) The other ones will take me a bit longer. Sorry. - -- Alexander berna...@gentoo.org https://secure.plaimi.net/~alexander -BEGIN PGP SIGNATURE- Version: GnuPG v2 iF4EAREIAAYFAlTjfHwACgkQRtClrXBQc7WAkwD/RxFKW0+T97eWGJRAJ/DK2GH3 NFOqxlucHmnKPdY5lsEBAJ7lqmHI5v3h3ojaVrDmbhbbeZC54jv9duDWGTt5Djte =xvS0 -END PGP SIGNATURE-
Re: [gentoo-portage-dev] [PATCH] binarytree: avoid unecessary index regeneration
-BEGIN PGP SIGNED MESSAGE- Hash: SHA512 On Tue, 17 Feb 2015 16:02:47 +0100 Alexander Berntsen berna...@gentoo.org wrote: -BEGIN PGP SIGNED MESSAGE- Hash: SHA256 LGTM. Go ahead push. - -- Alexander berna...@gentoo.org https://secure.plaimi.net/~alexander -BEGIN PGP SIGNATURE- Version: GnuPG v2 iF4EAREIAAYFAlTjWBYACgkQRtClrXBQc7V3CgEAjREJrZasgGrfP0fIdo7/oKZ7 /dMazIA2rnu6+CfeLlABAI9kc06NR2msTo4a1XTq6dxzSIrgqXIYxBfy/oDNY+rh =Fl7x -END PGP SIGNATURE- SURE! just reply to the EASY one ;) - -- Brian Dolbec dolsen -BEGIN PGP SIGNATURE- Version: GnuPG v2.0 iQJ8BAEBCgBmBQJU43u3XxSAAC4AKGlzc3Vlci1mcHJAbm90YXRpb25zLm9w ZW5wZ3AuZmlmdGhob3JzZW1hbi5uZXRBNUQ3Qzc0RTA4MUNDNzBEQjRBNEFBRjVG QkJEMDg3Mjc1ODIwRUQ4AAoJEPu9CHJ1gg7YPZ0P/1qdw/OA63FNps/gJPXJ0/Px m5W1uNoXFir4M9NYRGY4LxArPxskFUz7tPNFTzIe+SgcxoZ1z570RN/Yo8jf6Tbh rbdVn4aNXh5Tmc1zvPPQGX8CuMDHstoDhJKTKz/BtmVXn/ozsmSU1R8TnYZvVayc KpfetM9yYMlS51FU7SrI0jxWQ3hdpklnlcAM6/5DU920bM6Klhj04icF8MPD01kN +nVrdTPFN+jd0drn7pPiFASUUDRxZaVzjoz3TVQ7nSGQCTNLmzesmc2OFKJGbuzE ZMojnGGFMi/6X4TabKU9F9Z0hTkZRCTzUVD914ZXp16w3CBGNLMIDknVGMPS6VAf VhOytqzOuX1cvaHDtPRgFsfcfFXsFR5kbJJrTA8vwsxSf6f2mteycTYyxavhH86j pl5pQERPik4DRHGoIf2ir7NjgOuv/0z2rTjWEo8WRlfzi02QpqyiGnekbTB470M8 yFLjqQfcciCcd3NKmhIAh2J03ysPSQI1Y0SL/7zmBumlfAJefvtediKQtbAVyCT9 20ii3zawMyweWNGH78t9XXtyHs428jE91GOWZLgN4JVpaRcXp7WkJRN60LdpIaQD OU+jtA5P5efNTeHyPa0tj1LrGn2LCek8hys/4Hl0/xKzIM+uTxKQbkAqfVgQ+v4V hDXyB26lUNslv9PONvv8 =+r0T -END PGP SIGNATURE-
Re: [gentoo-portage-dev] [PATCH 1/2] Add FEATURES=binpkg-multi-instance (bug 150031)
On Tue, 17 Feb 2015 00:37:12 -0800
Zac Medico zmed...@gentoo.org wrote:
FEATURES=binpkg-multi-instance causes an integer build-id to be
associated with each binary package instance. Inclusion of the
build-id in the file name of the binary package file makes it
possible to store an arbitrary number of binary packages built from
the same ebuild.
Having multiple instances is useful for a number of purposes, such as
retaining builds that were built with different USE flags or linked
against different versions of libraries. The location of any
particular package within PKGDIR can be expressed as follows:
${PKGDIR}/${CATEGORY}/${PN}/${PF}-${BUILD_ID}.xpak
The build-id starts at 1 for the first build of a particular ebuild,
and is incremented by 1 for each new build. It is possible to share a
writable PKGDIR over NFS, and locking ensures that each package added
to PKGDIR will have a unique build-id. It is not necessary to migrate
an existing PKGDIR to the new layout, since portage is capable of
working with a mixed PKGDIR layout, where packages using the old
layout are allowed to remain in place.
The new PKGDIR layout is backward-compatible with binhost clients
running older portage, since the file format is identical, the
per-package PATH attribute in the 'Packages' index directs them to
download the file from the correct URI, and they automatically use
BUILD_TIME metadata to select the latest builds.
There is currently no automated way to prune old builds from PKGDIR,
although it is possible to remove packages manually, and then run
'emaint --fix binhost' to update the ${PKGDIR}/Packages index.
It is not necessary to migrate an existing PKGDIR to the new layout,
since portage is capable of working with a mixed PKGDIR layout, where
packages using the old layout are allowed to remain in-place.
There is currently no automated way to prune old builds from PKGDIR,
although it is possible to remove packages manually, and then run
'emaint --fix binhost' update the ${PKGDIR}/Packages index. Support
for FEATURES=binpkg-multi-instance is planned for eclean-pkg.
X-Gentoo-Bug: 150031
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=150031
---
bin/quickpkg | 1 -
man/make.conf.5| 27 +
pym/_emerge/Binpkg.py | 33 +-
pym/_emerge/BinpkgFetcher.py | 13 +-
pym/_emerge/BinpkgVerifier.py | 6 +-
pym/_emerge/EbuildBinpkg.py| 9 +-
pym/_emerge/EbuildBuild.py | 36 +-
pym/_emerge/Package.py | 67 +-
pym/_emerge/Scheduler.py | 6 +-
pym/_emerge/clear_caches.py| 1 -
pym/_emerge/resolver/output.py | 21 +-
pym/portage/const.py | 2 +
pym/portage/dbapi/__init__.py | 10 +-
pym/portage/dbapi/bintree.py | 842
+++--
pym/portage/dbapi/vartree.py | 8 +-
pym/portage/dbapi/virtual.py | 113 ++-
pym/portage/emaint/modules/binhost/binhost.py | 47 +-
pym/portage/package/ebuild/config.py | 3 +-
pym/portage/tests/resolver/ResolverPlayground.py | 26
+- .../resolver/binpkg_multi_instance/__init__.py | 2
+ .../resolver/binpkg_multi_instance/__test__.py | 2
+ .../binpkg_multi_instance/test_rebuilt_binaries.py | 101 +++
pym/portage/versions.py| 48 +- 23 files
changed, 932 insertions(+), 492 deletions(-) create mode 100644
overall, there is no way I know the code well enough to know if you
screwed up. But the code looks decent, so...
My only questions are:
pym/portage/dbapi/bintree.py:
You removed several functions from the binarytree class and essentially
reduced prevent_collision to a warning message. Can you briefly say
why they are not needed please.
_populate() == it is some 500+ LOC nasty. From what I can see I
think you added slightly more loc than you deleted/changed. While I
don't expect a breakup of this function to be directly a part of
this commit.
IT IS BADLY NEEDED.
I'd like to see this function properly split into pieces and act as
a driver for the smaller tasks. Currently it is a mess of nested
if/else, for,... that is extremely difficult to keep straight and
make logic changes to. It's almost as bad as repomans 1000+ LOC
main loop. Why if TRUE: == just fix the indent... At the same
time, I noticed _ensure_dirs, _file_permissions functions defined in
that class. Shouldn't these just be util functions available
everywhere.
pym/portage/versions.py
class _pkg_str(_unicode)
in __init__()
if build_time is not None:
try:
Re: [gentoo-dev] Re: [gentoo-commits] gentoo-x86 commit in dev-libs/libusbhp: ChangeLog Manifest libusbhp-1.0.2.ebuild metadata.xml
On 02/16/2015 13:01, Rich Freeman wrote: On Mon, Feb 16, 2015 at 11:02 AM, Joshua Kinard ku...@gentoo.org wrote: On 02/16/2015 09:04, Rich Freeman wrote: I do think that moving to a cleaner policy makes a lot of sense. The problem is that doing this sort of thing right potentially involves a lot of work as well. Maybe another approach is to just ditch per-file copyrights entirely (which a random perusal suggests is how Linux does things), but that would STILL require stripping the copyright out of these files with all the issues that entails, and limit our ability to borrow license-compatible code. Focusing on the last paragraph here (but not snipping), my understanding is the kernel retains per-file copyrights. This is why the kernel is permanently wedded to GPLv2, because some of the contributors owning those copyrights have died and thus can no longer consent to changing to the GPLv3 (or any other OSI license, or copyright change). Trying to track down their appropriate heirs, explain the whole situation, and then seek a consent would be a near-impossible undertaking. Hence, permanent GPLv2. Perhaps I should have worded that better. s/per-file copyrights/per-file copyright notices/ Obviously the content of individual files will always be copyrighted absent a release into the public domain. The Linux kernel just doesn't stick notices on individual files that attempt to identify who owns the copyright on what. Presumably that also means that if they borrow a file from somewhere else they don't care to change the copyright notice that was already there (or somehow they manage to avoid the euthusiasm stirred up by removing said notices). Well, I just sent a patch upstream that adds a new RTC driver to the kernel, and I added copyright to myself and the guy that created the original driver that I based off of to the top of the source file (and its header). So that practice is still used, and akpm recently added it to -mm with no comment on any of the copyright bits, so I must've gotten part that right. It's probably left to the person writing the specific source file(s) on how they want to do copyright, as long as they stick to recognized norms and GPLv2. -- Joshua Kinard Gentoo/MIPS ku...@gentoo.org 4096R/D25D95E3 2011-03-28 The past tempts us, the present confuses us, the future frightens us. And our lives slip away, moment by moment, lost in that vast, terrible in-between. --Emperor Turhan, Centauri Republic
Re: [gentoo-portage-dev] [PATCH 2/2] Add profile-formats=build-id (bug 150031)
On Tue, 17 Feb 2015 00:37:13 -0800 Zac Medico zmed...@gentoo.org wrote: When profile-formats = build-id is enabled in layout.conf of the containing repository, a dependency atom in the profile can refer to a specific build, using the build-id that is assigned when FEATURES=binpkg-multi-instance is enabled. A build-id atom is identical to a version-specific atom, except that the version is followed by a hyphen and an integer build-id. With the build-id profile format, it is possible to assemble a system using specific builds of binary packages, as users of binary distros might be accustomed to. For example, an atom in the packages file can pull a specific build of a package into the @system set, and an atom in the package.keywords file can be used to modify the effective KEYWORDS of a specific build of a package. Refering to specific builds can be useful for a number of reasons. For example, if a particular build needs to undergo a large amount of testing in a complex environment in order to verify reliability, then it can be useful to lock a profile to a specific build that has been thoroughly tested. X-Gentoo-Bug: 150031 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=150031 --- man/portage.5 | 8 +- pym/_emerge/is_valid_package_atom.py | 5 +- pym/portage/_sets/ProfilePackageSet.py | 3 +- pym/portage/_sets/profiles.py | 3 +- pym/portage/dep/__init__.py| 35 +- .../package/ebuild/_config/KeywordsManager.py | 3 +- .../package/ebuild/_config/LocationsManager.py | 8 +- pym/portage/package/ebuild/_config/MaskManager.py | 21 +++- pym/portage/package/ebuild/_config/UseManager.py | 14 ++- pym/portage/package/ebuild/config.py | 15 ++- pym/portage/repository/config.py | 2 +- pym/portage/tests/dep/test_isvalidatom.py | 8 +- .../test_build_id_profile_format.py| 134 + pym/portage/util/__init__.py | 13 +- 14 files changed, 234 insertions(+), 38 deletions(-) create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py class Atom() if allow_repo is None: allow_repo = True + if allow_build_id is None: + allow_build_id = True these can be written as allow_repo = allow_repo or True allow_build_id = allow_build_id or True Otherwise looks decent. -- Brian Dolbec dolsen
Re: [gentoo-portage-dev] [PATCH 2/2] Add profile-formats=build-id (bug 150031)
On 02/17/2015 10:58 AM, Brian Dolbec wrote: class Atom() if allow_repo is None: allow_repo = True + if allow_build_id is None: + allow_build_id = True these can be written as allow_repo = allow_repo or True allow_build_id = allow_build_id or True Actually, your version behaves differently than mine for the case where False has been passed in for these parameters. The parameters are designed are provide a smart default, as long as the caller has not passed in an explicit True or False value. -- Thanks, Zac
Re: [gentoo-portage-dev] [PATCH 1/2] Add FEATURES=binpkg-multi-instance (bug 150031)
On 02/17/2015 11:56 AM, Brian Dolbec wrote: On Tue, 17 Feb 2015 11:26:27 -0800 Zac Medico zmed...@gentoo.org wrote: On 02/17/2015 10:42 AM, Brian Dolbec wrote: overall, there is no way I know the code well enough to know if you screwed up. But the code looks decent, so... My only questions are: pym/portage/dbapi/bintree.py: You removed several functions from the binarytree class and essentially reduced prevent_collision to a warning message. Can you briefly say why they are not needed please. Okay, I'll include this info in an updated patch: Actually, I think this one patch could be split into a few logical ones. Tag them binpkg-multi-instance 1 of... in the commit message so it is clear they belong together. That way the commit messages can more clearly be relevant to the file(s) changed. The commit message is already a short story in length ;) before adding these new explanations. Okay, will do. -- Thanks, Zac
[gentoo-portage-dev] [PATCH] binarytree: avoid unecessary index regeneration
Since commit f1c1b8a77eebf7713b32e5f9945690f60f4f46de, binarytree regenerates the 'Packages' index unnecessarily, due to missing REQUIRES and PROVIDES entries. These entries are not required, so use default empty values in order to avoid the unnecessary regeneration. Fixes: f1c1b8a77eeb (Generate soname dependency metadata (bug 282639)) --- pym/portage/dbapi/bintree.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index b98b26e..726a7a1 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -344,7 +344,9 @@ class binarytree(object): PDEPEND : , PROPERTIES : , PROVIDE : , + PROVIDES: , RDEPEND : , + REQUIRES: , RESTRICT: , SLOT: 0, USE : , -- 2.0.5
[gentoo-portage-dev] [PATCH 1/2] Add FEATURES=binpkg-multi-instance (bug 150031)
FEATURES=binpkg-multi-instance causes an integer build-id to be
associated with each binary package instance. Inclusion of the build-id
in the file name of the binary package file makes it possible to store
an arbitrary number of binary packages built from the same ebuild.
Having multiple instances is useful for a number of purposes, such as
retaining builds that were built with different USE flags or linked
against different versions of libraries. The location of any particular
package within PKGDIR can be expressed as follows:
${PKGDIR}/${CATEGORY}/${PN}/${PF}-${BUILD_ID}.xpak
The build-id starts at 1 for the first build of a particular ebuild,
and is incremented by 1 for each new build. It is possible to share a
writable PKGDIR over NFS, and locking ensures that each package added
to PKGDIR will have a unique build-id. It is not necessary to migrate
an existing PKGDIR to the new layout, since portage is capable of
working with a mixed PKGDIR layout, where packages using the old layout
are allowed to remain in place.
The new PKGDIR layout is backward-compatible with binhost clients
running older portage, since the file format is identical, the
per-package PATH attribute in the 'Packages' index directs them to
download the file from the correct URI, and they automatically use
BUILD_TIME metadata to select the latest builds.
There is currently no automated way to prune old builds from PKGDIR,
although it is possible to remove packages manually, and then run
'emaint --fix binhost' to update the ${PKGDIR}/Packages index.
It is not necessary to migrate an existing PKGDIR to the new layout,
since portage is capable of working with a mixed PKGDIR layout, where
packages using the old layout are allowed to remain in-place.
There is currently no automated way to prune old builds from PKGDIR,
although it is possible to remove packages manually, and then run
'emaint --fix binhost' update the ${PKGDIR}/Packages index. Support for
FEATURES=binpkg-multi-instance is planned for eclean-pkg.
X-Gentoo-Bug: 150031
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=150031
---
bin/quickpkg | 1 -
man/make.conf.5| 27 +
pym/_emerge/Binpkg.py | 33 +-
pym/_emerge/BinpkgFetcher.py | 13 +-
pym/_emerge/BinpkgVerifier.py | 6 +-
pym/_emerge/EbuildBinpkg.py| 9 +-
pym/_emerge/EbuildBuild.py | 36 +-
pym/_emerge/Package.py | 67 +-
pym/_emerge/Scheduler.py | 6 +-
pym/_emerge/clear_caches.py| 1 -
pym/_emerge/resolver/output.py | 21 +-
pym/portage/const.py | 2 +
pym/portage/dbapi/__init__.py | 10 +-
pym/portage/dbapi/bintree.py | 842 +++--
pym/portage/dbapi/vartree.py | 8 +-
pym/portage/dbapi/virtual.py | 113 ++-
pym/portage/emaint/modules/binhost/binhost.py | 47 +-
pym/portage/package/ebuild/config.py | 3 +-
pym/portage/tests/resolver/ResolverPlayground.py | 26 +-
.../resolver/binpkg_multi_instance/__init__.py | 2 +
.../resolver/binpkg_multi_instance/__test__.py | 2 +
.../binpkg_multi_instance/test_rebuilt_binaries.py | 101 +++
pym/portage/versions.py| 48 +-
23 files changed, 932 insertions(+), 492 deletions(-)
create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/__init__.py
create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/__test__.py
create mode 100644
pym/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
diff --git a/bin/quickpkg b/bin/quickpkg
index 2c69a69..8b71c3e 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -63,7 +63,6 @@ def quickpkg_atom(options, infos, arg, eout):
pkgs_for_arg = 0
for cpv in matches:
excluded_config_files = []
- bintree.prevent_collision(cpv)
dblnk = vardb._dblink(cpv)
have_lock = False
diff --git a/man/make.conf.5 b/man/make.conf.5
index 84b7191..6ead61b 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -256,6 +256,33 @@ has a \fB\-\-force\fR option that can be used to force
regeneration of digests.
Keep logs from successful binary package merges. This is relevant only when
\fBPORT_LOGDIR\fR is set.
.TP
+.B binpkg\-multi\-instance
+Enable support for multiple binary package instances per ebuild.
+Having multiple instances is useful for a number of purposes, such as
+retaining builds that were built with different USE flags or linked
+against different versions of libraries. The location of any particular
+package within PKGDIR can be expressed as follows:
+
+ ${PKGDIR}/${CATEGORY}/${PN}/${PF}\-${BUILD_ID}.xpak
+
[gentoo-portage-dev] [PATCH 2/2] Add profile-formats=build-id (bug 150031)
When profile-formats = build-id is enabled in layout.conf of the containing repository, a dependency atom in the profile can refer to a specific build, using the build-id that is assigned when FEATURES=binpkg-multi-instance is enabled. A build-id atom is identical to a version-specific atom, except that the version is followed by a hyphen and an integer build-id. With the build-id profile format, it is possible to assemble a system using specific builds of binary packages, as users of binary distros might be accustomed to. For example, an atom in the packages file can pull a specific build of a package into the @system set, and an atom in the package.keywords file can be used to modify the effective KEYWORDS of a specific build of a package. Refering to specific builds can be useful for a number of reasons. For example, if a particular build needs to undergo a large amount of testing in a complex environment in order to verify reliability, then it can be useful to lock a profile to a specific build that has been thoroughly tested. X-Gentoo-Bug: 150031 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=150031 --- man/portage.5 | 8 +- pym/_emerge/is_valid_package_atom.py | 5 +- pym/portage/_sets/ProfilePackageSet.py | 3 +- pym/portage/_sets/profiles.py | 3 +- pym/portage/dep/__init__.py| 35 +- .../package/ebuild/_config/KeywordsManager.py | 3 +- .../package/ebuild/_config/LocationsManager.py | 8 +- pym/portage/package/ebuild/_config/MaskManager.py | 21 +++- pym/portage/package/ebuild/_config/UseManager.py | 14 ++- pym/portage/package/ebuild/config.py | 15 ++- pym/portage/repository/config.py | 2 +- pym/portage/tests/dep/test_isvalidatom.py | 8 +- .../test_build_id_profile_format.py| 134 + pym/portage/util/__init__.py | 13 +- 14 files changed, 234 insertions(+), 38 deletions(-) create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py diff --git a/man/portage.5 b/man/portage.5 index ed5140d..e062f9f 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -1180,7 +1180,7 @@ and the newer/faster md5-dict format. Default is to detect dirs. The EAPI to use for profiles when unspecified. This attribute is supported only if profile-default-eapi is included in profile-formats. .TP -.BR profile\-formats = [pms] [portage-1] [portage-2] [profile-bashrcs] [profile-set] [profile-default-eapi] +.BR profile\-formats = [pms] [portage-1] [portage-2] [profile-bashrcs] [profile-set] [profile-default-eapi] [build-id] Control functionality available to profiles in this repo such as which files may be dirs, or the syntax available in parent files. Use portage-2 if you're unsure. The default is portage-1-compat mode which is meant to be compatible @@ -1190,7 +1190,11 @@ Setting profile-bashrcs will enable the per-profile bashrc mechanism profile \fBpackages\fR file to add atoms to the @profile package set. See the profile \fBpackages\fR section for more information. Setting profile-default-eapi enables support for the -profile_eapi_when_unspecified attribute. +profile_eapi_when_unspecified attribute. Setting build\-id allows +dependency atoms in the profile to refer to specific builds (see the +binpkg\-multi\-instance FEATURES setting in \fBmake.conf\fR(5)). A +build\-id atom is identical to a version-specific atom, except that the +version is followed by a hyphen and an integer build\-id. .RE .RE diff --git a/pym/_emerge/is_valid_package_atom.py b/pym/_emerge/is_valid_package_atom.py index 112afc1..17f7642 100644 --- a/pym/_emerge/is_valid_package_atom.py +++ b/pym/_emerge/is_valid_package_atom.py @@ -14,9 +14,10 @@ def insert_category_into_atom(atom, category): ret = None return ret -def is_valid_package_atom(x, allow_repo=False): +def is_valid_package_atom(x, allow_repo=False, allow_build_id=True): if / not in x.split(:)[0]: x2 = insert_category_into_atom(x, 'cat') if x2 != None: x = x2 - return isvalidatom(x, allow_blockers=False, allow_repo=allow_repo) + return isvalidatom(x, allow_blockers=False, allow_repo=allow_repo, + allow_build_id=allow_build_id) diff --git a/pym/portage/_sets/ProfilePackageSet.py b/pym/portage/_sets/ProfilePackageSet.py index 2fcafb6..fec9373 100644 --- a/pym/portage/_sets/ProfilePackageSet.py +++ b/pym/portage/_sets/ProfilePackageSet.py @@ -23,7 +23,8 @@ class ProfilePackageSet(PackageSet): def load(self): self._setAtoms(x for x in stack_lists( [grabfile_package(os.path.join(y.location, packages), - verify_eapi=True, eapi=y.eapi, eapi_default=None) + verify_eapi=True,
