[gentoo-dev] Last rites: sci-mathematics/ggnfs
# Sam James (2021-02-27) # Fails to build with GCC 10 (or otherwise!) # bug #708508, bug #728026, bug #542280 # Removal in 30 days sci-mathematics/ggnfs signature.asc Description: Message signed with OpenPGP
[gentoo-dev] Last rites: dev-lang/ats
# Sam James (2021-02-27) # Fails to build with GCC 10, out of date. # bug #723192, bug #737058 # Removal in 30 days dev-lang/ats signature.asc Description: Message signed with OpenPGP
[gentoo-dev] Last rites: dev-scheme/greg
# Sam James (2021-02-27) # Broken with newer(?) dev-scheme/guile, dead upstream # bug #642736, bug #773196 dev-scheme/greg signature.asc Description: Message signed with OpenPGP
[gentoo-portage-dev] [PATCH v3] emirrordist: add --content-db option required for content-hash layout (bug 756778)
Add a --content-db option which is required for the content-hash
layout because its file listings return content digests instead of
distfile names.
The content db serves to translate content digests to distfiles
names, and distfiles names to content digests. All keys have a
prefix separated by a colon. For digest keys, the prefix is the
hash algorithm name. For filename keys, the prefix is "filename".
The value associated with a digest key is a set of file names. The
value associated with a distfile key is a set of content revisions.
Each content revision is expressed as a dictionary of digests which
is suitable for construction of a DistfileName instance.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico
---
[PATCH v3] changed the value associated with a digest key is a set
of file name, and fixed ContentDB.remove to preserved independent
references to identical content (like removing one of multiple
hardlinks).
lib/portage/_emirrordist/Config.py | 8 +-
lib/portage/_emirrordist/ContentDB.py| 178 +++
lib/portage/_emirrordist/DeletionIterator.py | 25 ++-
lib/portage/_emirrordist/DeletionTask.py | 8 +
lib/portage/_emirrordist/FetchTask.py| 5 +-
lib/portage/_emirrordist/main.py | 15 +-
lib/portage/package/ebuild/fetch.py | 8 +-
lib/portage/tests/ebuild/test_fetch.py | 14 ++
man/emirrordist.1| 6 +-
9 files changed, 256 insertions(+), 11 deletions(-)
create mode 100644 lib/portage/_emirrordist/ContentDB.py
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index 4bee4f45e..cfe944040 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -10,6 +10,7 @@ import time
from portage import os
from portage.package.ebuild.fetch import MirrorLayoutConfig
from portage.util import grabdict, grablines
+from .ContentDB import ContentDB
class Config:
def __init__(self, options, portdb, event_loop):
@@ -65,6 +66,11 @@ class Config:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
+ self.content_db = None
+ if options.content_db is not None:
+ self.content_db = ContentDB(self._open_shelve(
+ options.content_db, 'content'))
+
self.deletion_db = None
if options.deletion_db is not None:
self.deletion_db = self._open_shelve(
diff --git a/lib/portage/_emirrordist/ContentDB.py
b/lib/portage/_emirrordist/ContentDB.py
new file mode 100644
index 0..7084cecff
--- /dev/null
+++ b/lib/portage/_emirrordist/ContentDB.py
@@ -0,0 +1,178 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import operator
+import shelve
+import typing
+
+from portage.package.ebuild.fetch import DistfileName
+
+
+class ContentDB:
+ """
+ The content db serves to translate content digests to distfiles
+ names, and distfiles names to content digests. All keys have a
+ prefix separated by a colon. For digest keys, the prefix is the
+ hash algorithm name. For filename keys, the prefix is "filename".
+
+ The value associated with a digest key is a set of file names. The
+ value associated with a distfile key is a set of content revisions.
+ Each content revision is expressed as a dictionary of digests which
+ is suitable for construction of a DistfileName instance.
+ """
+
+ def __init__(self, shelve_instance: shelve.Shelf):
+ self._shelve = shelve_instance
+
+ def add(self, filename: DistfileName):
+ """
+ Add file name and digests.
+
+ @param filename: file name with digests attribute
+ """
+ distfile_str = str(filename)
+ distfile_key = "filename:{}".format(distfile_str)
+ for k, v in filename.digests.items():
+ if k != "size":
+ digest_key = "{}:{}".format(k, v).lower()
+ try:
+ digest_files = self._shelve[digest_key]
+ except KeyError:
+ digest_files = set()
+ digest_files.add(distfile_str)
+ self._shelve[digest_key] = digest_files
+ try:
+ content_revisions = self._shelve[distfile_key]
+ except KeyError:
+ content_revisions = set()
+
+ revision_key = tuple(
+
[gentoo-portage-dev] [PATCH] make.globals: make FEATURES=-binpkg-multi-instance sticky for existing installs
Add a _compat_upgrade.binpkg_multi_instance script that the ebuild
can call in pkg_preinst in order to maintain a backward-compatible
FEATURES=-binpkg-multi-instance default on existing installs where
enabling binpkg-multi-instance could cause disruption.
Bug: https://bugs.gentoo.org/772785
Signed-off-by: Zac Medico
---
.../_compat_upgrade/binpkg_multi_instance.py | 33 +++
1 file changed, 33 insertions(+)
create mode 100644 lib/portage/_compat_upgrade/binpkg_multi_instance.py
diff --git a/lib/portage/_compat_upgrade/binpkg_multi_instance.py
b/lib/portage/_compat_upgrade/binpkg_multi_instance.py
new file mode 100644
index 0..bfeb72baa
--- /dev/null
+++ b/lib/portage/_compat_upgrade/binpkg_multi_instance.py
@@ -0,0 +1,33 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import portage
+from portage import os
+from portage.const import GLOBAL_CONFIG_PATH
+
+COMPAT_FEATURES = 'FEATURES="${FEATURES} -binpkg-multi-instance"'
+
+
+def main():
+ """
+ If the current installation is still has binpkg-multi-instance
+ disabled, then patch make.globals inside ${ED} to maintain backward
+ compatibility. This is intended to be called from the ebuild as
+ follows:
+
+ pkg_preinst() {
+ python_setup
+ env -u FEATURES
+
PYTHONPATH="${D%/}$(python_get_sitedir)${PYTHONPATH:+:${PYTHONPATH}}" \
+ "${PYTHON}" -m
portage._compat_upgrade.binpkg_multi_instance || die
+ }
+ """
+ if 'binpkg-multi-instance' not in portage.settings.features:
+ portage.output.EOutput().einfo('Setting make.globals default {}
for backward compatibility'.format(COMPAT_FEATURES))
+ config_path = os.path.join(os.environ['ED'],
GLOBAL_CONFIG_PATH.lstrip(os.sep), 'make.globals')
+ with open(config_path, 'at') as f:
+ f.write("{}\n".format(COMPAT_FEATURES))
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
[gentoo-portage-dev] [PATCH v2] emirrordist: add --content-db option required for content-hash layout (bug 756778)
Add a --content-db option which is required for the content-hash
layout because its file listings return content digests instead of
distfile names.
The content db serves to translate content digests to distfiles
names, and distfiles names to content digests. All keys have a
prefix separated by a colon. For digest keys, the prefix is the
hash algorithm name. For filename keys, the prefix is "filename".
The value associated with a digest key is a plain filename. The
value associated with a distfile key is a set of content revisions.
Each content revision is expressed as a dictionary of digests which
is suitable for construction of a DistfileName instance.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico
---
[PATCH v2] Split out ContentDB class and associate distfile key
with a set of content revisions, where each content revision is
expressed as a dictionary of digests.
lib/portage/_emirrordist/Config.py | 8 +-
lib/portage/_emirrordist/ContentDB.py| 158 +++
lib/portage/_emirrordist/DeletionIterator.py | 25 ++-
lib/portage/_emirrordist/DeletionTask.py | 8 +
lib/portage/_emirrordist/FetchTask.py| 5 +-
lib/portage/_emirrordist/main.py | 15 +-
lib/portage/tests/ebuild/test_fetch.py | 14 ++
man/emirrordist.1| 6 +-
8 files changed, 232 insertions(+), 7 deletions(-)
create mode 100644 lib/portage/_emirrordist/ContentDB.py
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index 4bee4f45e..cfe944040 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -10,6 +10,7 @@ import time
from portage import os
from portage.package.ebuild.fetch import MirrorLayoutConfig
from portage.util import grabdict, grablines
+from .ContentDB import ContentDB
class Config:
def __init__(self, options, portdb, event_loop):
@@ -65,6 +66,11 @@ class Config:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
+ self.content_db = None
+ if options.content_db is not None:
+ self.content_db = ContentDB(self._open_shelve(
+ options.content_db, 'content'))
+
self.deletion_db = None
if options.deletion_db is not None:
self.deletion_db = self._open_shelve(
diff --git a/lib/portage/_emirrordist/ContentDB.py
b/lib/portage/_emirrordist/ContentDB.py
new file mode 100644
index 0..60e6ef39d
--- /dev/null
+++ b/lib/portage/_emirrordist/ContentDB.py
@@ -0,0 +1,158 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import operator
+import shelve
+import typing
+
+from portage.package.ebuild.fetch import DistfileName
+
+
+class ContentDB:
+ """
+ The content db serves to translate content digests to distfiles
+ names, and distfiles names to content digests. All keys have a
+ prefix separated by a colon. For digest keys, the prefix is the
+ hash algorithm name. For filename keys, the prefix is "filename".
+
+ The value associated with a digest key is a plain filename. The
+ value associated with a distfile key is a set of content revisions.
+ Each content revision is expressed as a dictionary of digests which
+ is suitable for construction of a DistfileName instance.
+ """
+
+ def __init__(self, shelve_instance: shelve.Shelf):
+ self._shelve = shelve_instance
+
+ def add(self, filename: DistfileName):
+ """
+ Add file name and digests.
+
+ @param filename: file name with digests attribute
+ """
+ distfile_str = str(filename)
+ distfile_key = "filename:{}".format(distfile_str)
+ for k, v in filename.digests.items():
+ if k != "size":
+ self._shelve["{}:{}".format(k, v).lower()] =
distfile_str
+ try:
+ content_revisions = self._shelve[distfile_key]
+ except KeyError:
+ content_revisions = set()
+
+ revision_key = tuple(
+ sorted(
+ (
+ (algo.lower(),
filename.digests[algo].lower())
+ for algo in filename.digests
+ if algo != "size"
+ ),
+ key=operator.itemgetter(0),
+ )
+ )
+
