[gentoo-portage-dev] [PATCH 1/2] Add get_repo_revision_history function and repo_revisions file
The history of synced revisions is provided by a new
get_repo_revision_history function and corresponding
/var/lib/portage/repo_revisions file, with history
limit currently capped at 25 revisions. If a change
is detected and the current process has permission
to update the repo_revisions file, then the file will
be updated with any newly detected revisions.
For volatile repos the revisions may be unordered,
which makes them unusable for the purposes of the
revision history, so the revisions of volatile repos
are not tracked. This functions detects revisions
which are not yet visible to the current process due
to the sync-rcu option.
The emaint revisions --purgerepos and --purgeallrepos
options allow revisions for some or all repos to be
easily purged from the history. For example, the
emerge-webrsync script uses this emaint commmand to
purge the revision history of the gentoo repo when
the emerge-webrsync --revert option is used to roll
back to a previous snapshot:
emaint revisions --purgerepos="${repo_name}"
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico
---
bin/emerge-webrsync | 3 +-
lib/portage/const.py | 1 +
lib/portage/emaint/modules/meson.build| 1 +
.../emaint/modules/revisions/__init__.py | 36 +
.../emaint/modules/revisions/meson.build | 8 ++
.../emaint/modules/revisions/revisions.py | 95 +
lib/portage/sync/controller.py| 8 +-
lib/portage/sync/meson.build | 1 +
lib/portage/sync/revision_history.py | 133 ++
lib/portage/tests/sync/test_sync_local.py | 75 +-
man/emaint.1 | 18 ++-
man/portage.5 | 17 ++-
12 files changed, 388 insertions(+), 8 deletions(-)
create mode 100644 lib/portage/emaint/modules/revisions/__init__.py
create mode 100644 lib/portage/emaint/modules/revisions/meson.build
create mode 100644 lib/portage/emaint/modules/revisions/revisions.py
create mode 100644 lib/portage/sync/revision_history.py
diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync
index 99da05543a..caa4986da2 100755
--- a/bin/emerge-webrsync
+++ b/bin/emerge-webrsync
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# Author: Karl Trygve Kalleberg
# Rewritten from the old, Perl-based emerge-webrsync script
@@ -732,6 +732,7 @@ main() {
[[ ${do_debug} -eq 1 ]] && set -x
if [[ -n ${revert_date} ]] ; then
+ emaint revisions --purgerepos="${repo_name}"
do_snapshot 1 "${revert_date}"
else
do_latest_snapshot
diff --git a/lib/portage/const.py b/lib/portage/const.py
index 2154213b7b..c9a71009a7 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -51,6 +51,7 @@ PRIVATE_PATH = "var/lib/portage"
WORLD_FILE = f"{PRIVATE_PATH}/world"
WORLD_SETS_FILE = f"{PRIVATE_PATH}/world_sets"
CONFIG_MEMORY_FILE = f"{PRIVATE_PATH}/config"
+REPO_REVISIONS = f"{PRIVATE_PATH}/repo_revisions"
NEWS_LIB_PATH = "var/lib/gentoo"
# these variables get EPREFIX prepended automagically when they are
diff --git a/lib/portage/emaint/modules/meson.build
b/lib/portage/emaint/modules/meson.build
index 48f4f77d83..33b396be94 100644
--- a/lib/portage/emaint/modules/meson.build
+++ b/lib/portage/emaint/modules/meson.build
@@ -12,5 +12,6 @@ subdir('logs')
subdir('merges')
subdir('move')
subdir('resume')
+subdir('revisions')
subdir('sync')
subdir('world')
diff --git a/lib/portage/emaint/modules/revisions/__init__.py
b/lib/portage/emaint/modules/revisions/__init__.py
new file mode 100644
index 00..c51cbb4bf3
--- /dev/null
+++ b/lib/portage/emaint/modules/revisions/__init__.py
@@ -0,0 +1,36 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+doc = """Purge repo_revisions history file."""
+__doc__ = doc
+
+
+module_spec = {
+"name": "revisions",
+"description": doc,
+"provides": {
+"purgerevisions": {
+"name": "revisions",
+"sourcefile": "revisions",
+"class": "PurgeRevisions",
+"description": "Purge repo_revisions history",
+"functions": ["purgeallrepos", "purgerepos"],
+"func_desc": {
+"repo": {
+"long": "--purgerepos",
+"help": "(revisions module only): --purgerepos Purge
revisions for the specif
[gentoo-portage-dev] [PATCH 2/2] bintree: Add REPO_REVISIONS to package index header
As a means for binhost clients to select source repo
revisions which are consistent with binhosts, inject
REPO_REVISIONS from a package into the index header,
using a history of synced revisions to guarantee
forward progress. This queries the relevant repos to
check if any new revisions have appeared in the
absence of a proper sync operation.
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico
---
lib/portage/dbapi/bintree.py | 66 -
lib/portage/tests/sync/test_sync_local.py | 71 +++
2 files changed, 123 insertions(+), 14 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 7bc1f60f6d..fbf60e74eb 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -48,6 +48,7 @@ from portage.exception import (
from portage.localization import _
from portage.output import colorize
from portage.package.ebuild.profile_iuse import iter_iuse_vars
+from portage.sync.revision_history import get_repo_revision_history
from portage.util import ensure_dirs
from portage.util.file_copy import copyfile
from portage.util.futures import asyncio
@@ -62,6 +63,7 @@ from portage import _unicode_encode
import codecs
import errno
import io
+import json
import re
import stat
import subprocess
@@ -134,13 +136,19 @@ class bindbapi(fakedbapi):
"USE",
"_mtime_",
}
+# Keys required only when initially adding a package.
+self._init_aux_keys = {
+"REPO_REVISIONS",
+}
self._aux_cache = {}
self._aux_cache_slot_dict_cache = None
@property
def _aux_cache_slot_dict(self):
if self._aux_cache_slot_dict_cache is None:
-self._aux_cache_slot_dict_cache =
slot_dict_class(self._aux_cache_keys)
+self._aux_cache_slot_dict_cache = slot_dict_class(
+chain(self._aux_cache_keys, self._init_aux_keys)
+)
return self._aux_cache_slot_dict_cache
def __getstate__(self):
@@ -1791,6 +1799,10 @@ class binarytree:
pkgindex = self._new_pkgindex()
d = self._inject_file(pkgindex, cpv, full_path)
+repo_revisions = d.get("REPO_REVISIONS")
+if repo_revisions:
+repo_revisions = json.loads(repo_revisions)
+self._inject_repo_revisions(pkgindex.header, repo_revisions)
self._update_pkgindex_header(pkgindex.header)
self._pkgindex_write(pkgindex)
@@ -1872,7 +1884,7 @@ class binarytree:
@return: package metadata
"""
if keys is None:
-keys = self.dbapi._aux_cache_keys
+keys = chain(self.dbapi._aux_cache_keys, self.dbapi._init_aux_keys)
metadata = self.dbapi._aux_cache_slot_dict()
else:
metadata = {}
@@ -1916,6 +1928,56 @@ class binarytree:
return metadata
+def _inject_repo_revisions(self, header, repo_revisions):
+"""
+Inject REPO_REVISIONS from a package into the index header,
+using a history of synced revisions to guarantee forward
+progress. This queries the relevant repos to check if any
+new revisions have appeared in the absence of a proper sync
+operation.
+
+This does not expose REPO_REVISIONS that do not appear in
+the sync history, since such revisions suggest that the
+package was not built locally, and in this case its
+REPO_REVISIONS are not intended to be exposed.
+"""
+synced_repo_revisions = get_repo_revision_history(
+self.settings["EROOT"],
+[self.settings.repositories[repo_name] for repo_name in
repo_revisions],
+)
+header_repo_revisions = (
+json.loads(header["REPO_REVISIONS"]) if
header.get("REPO_REVISIONS") else {}
+)
+for repo_name, repo_revision in repo_revisions.items():
+rev_list = synced_repo_revisions.get(repo_name, [])
+header_rev = header_repo_revisions.get(repo_name)
+if not rev_list or header_rev in (repo_revision, rev_list[0]):
+continue
+try:
+header_rev_index = (
+None if header_rev is None else rev_list.index(header_rev)
+)
+except ValueError:
+header_rev_index = None
+try:
+repo_revision_index = rev_list.index(repo_revision)
+except ValueError:
+repo_revision_index = None
+if repo_revision_index is not None and (
+header_rev_index is None or repo_revision_index <
header_rev_index
+):
+# There is forward progress when repo_revision is more recent
+# than header_rev
[gentoo-portage-dev] [PATCH 0/2] Add REPO_REVISIONS to package index header
As a means for binhost clients to select source repo
revisions which are consistent with binhosts, inject
REPO_REVISIONS from a package into the index header,
using a history of synced revisions to guarantee
forward progress. This queries the relevant repos to
check if any new revisions have appeared in the
absence of a proper sync operation.
The history of synced revisions is provided by a new
get_repo_revision_history function and corresponding
/var/lib/portage/repo_revisions file, with history
limit currently capped at 25 revisions. If a change
is detected and the current process has permission
to update the repo_revisions file, then the file will
be updated with any newly detected revisions.
For volatile repos the revisions may be unordered,
which makes them unusable for the purposes of the
revision history, so the revisions of volatile repos
are not tracked. This functions detects revisions
which are not yet visible to the current process due
to the sync-rcu option.
The emaint revisions --purgerepos and --purgeallrepos
options allow revisions for some or all repos to be
easily purged from the history. For example, the
emerge-webrsync script uses this emaint commmand to
purge the revision history of the gentoo repo when
the emerge-webrsync --revert option is used to roll
back to a previous snapshot:
emaint revisions --purgerepos="${repo_name}"
This series can also be reviewed at https://github.com/gentoo/portage/pull/1306.
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico
Zac Medico (2):
Add get_repo_revision_history function and repo_revisions file
bintree: Add REPO_REVISIONS to package index header
bin/emerge-webrsync | 3 +-
lib/portage/const.py | 1 +
lib/portage/dbapi/bintree.py | 66 +++-
lib/portage/emaint/modules/meson.build| 1 +
.../emaint/modules/revisions/__init__.py | 36 +
.../emaint/modules/revisions/meson.build | 8 +
.../emaint/modules/revisions/revisions.py | 95
lib/portage/sync/controller.py| 8 +-
lib/portage/sync/meson.build | 1 +
lib/portage/sync/revision_history.py | 133
lib/portage/tests/sync/test_sync_local.py | 146 --
man/emaint.1 | 18 ++-
man/portage.5 | 17 +-
13 files changed, 511 insertions(+), 22 deletions(-)
create mode 100644 lib/portage/emaint/modules/revisions/__init__.py
create mode 100644 lib/portage/emaint/modules/revisions/meson.build
create mode 100644 lib/portage/emaint/modules/revisions/revisions.py
create mode 100644 lib/portage/sync/revision_history.py
--
2.41.0
[gentoo-portage-dev] [PATCH 9/9] EbuildPhase: async_check_locale
Change config.environ() check_locale calls to async_check_locale
calls in the EbuildPhase _async_start method in order to eliminate
synchronous waiting for child processes in the main event loop
thread.
Bug: https://bugs.gentoo.org/923841
Signed-off-by: Zac Medico
---
lib/_emerge/EbuildMetadataPhase.py| 4 +++
lib/_emerge/EbuildPhase.py| 28 ++-
lib/portage/package/ebuild/config.py | 26 -
lib/portage/util/futures/_asyncio/__init__.py | 9 ++
lib/portage/util/locale.py| 28 +--
5 files changed, 70 insertions(+), 25 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index 9fcdabe840..90a3ea05aa 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,6 +8,7 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"_emerge.EbuildPhase:_setup_locale",
"portage.package.ebuild._metadata_invalid:eapi_invalid",
)
from portage import os
@@ -83,6 +84,9 @@ class EbuildMetadataPhase(SubProcess):
settings.setcpv(self.cpv)
settings.configdict["pkg"]["EAPI"] = parsed_eapi
+# This requires above setcpv and EAPI setup.
+await _setup_locale(self.settings)
+
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index c81bf54a81..c8caf73722 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import (
_prepare_fake_distdir,
_prepare_fake_filesdir,
)
+from portage.eapi import _get_eapi_attrs
from portage.util import writemsg, ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport(
+ "_post_src_install_write_metadata,"
+ "_preinst_bsdflags",
"portage.util.futures.unix_events:_set_nonblocking",
+"portage.util.locale:async_check_locale,split_LC_ALL",
)
from portage import os
from portage import _encodings
from portage import _unicode_encode
+async def _setup_locale(settings):
+eapi_attrs = _get_eapi_attrs(settings["EAPI"])
+if eapi_attrs.posixish_locale:
+split_LC_ALL(settings)
+settings["LC_COLLATE"] = "C"
+# check_locale() returns None when check can not be executed.
+if await async_check_locale(silent=True, env=settings.environ()) is
False:
+# try another locale
+for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
+settings["LC_CTYPE"] = l
+if await async_check_locale(silent=True,
env=settings.environ()):
+# TODO: output the following only once
+# writemsg(
+# _("!!! LC_CTYPE unsupported, using %s instead\n")
+# % self.settings["LC_CTYPE"]
+# )
+break
+else:
+raise AssertionError("C locale did not pass the test!")
+
+
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") +
("_ebuild_lock",)
@@ -94,6 +117,9 @@ class EbuildPhase(CompositeTask):
self._start_task(AsyncTaskFuture(future=future),
self._async_start_exit)
async def _async_start(self):
+
+await _setup_locale(self.settings)
+
need_builddir = self.phase not in
EbuildProcess._phases_without_builddir
if need_builddir:
diff --git a/lib/portage/package/ebuild/config.py
b/lib/portage/package/ebuild/config.py
index d7b0ca5676..35c77486ec 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -29,7 +29,6 @@ portage.proxy.lazyimport.lazyimport(
"portage.dbapi.vartree:vartree",
"portage.package.ebuild.doebuild:_phase_func_map",
"portage.util.compression_probe:_compressors",
-"portage.util.locale:check_locale,split_LC_ALL",
)
from portage import bsd_chflags, load_mod, os, selinux, _unicode_decode
from portage.const import (
@@ -3368,20 +3367,17 @@ class config:
mydict["EBUILD_PHASE_FUNC"] = phase_func
if eapi_attrs.posixish_locale:
-split_LC_ALL(mydict)
-mydict["LC_COLLATE"] = "C"
-
[gentoo-portage-dev] [PATCH 8/9] EbuildMetadataPhase: Migrate to _async_start
Bug: https://bugs.gentoo.org/923841 Signed-off-by: Zac Medico --- lib/_emerge/EbuildMetadataPhase.py | 17 ++--- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/_emerge/EbuildMetadataPhase.py b/lib/_emerge/EbuildMetadataPhase.py index 784712e8cb..9fcdabe840 100644 --- a/lib/_emerge/EbuildMetadataPhase.py +++ b/lib/_emerge/EbuildMetadataPhase.py @@ -46,6 +46,12 @@ class EbuildMetadataPhase(SubProcess): _files_dict = slot_dict_class(_file_names, prefix="") def _start(self): +asyncio.ensure_future( +self._async_start(), loop=self.scheduler +).add_done_callback(self._async_start_done) +self._registered = True + +async def _async_start(self): ebuild_path = self.ebuild_hash.location with open( @@ -116,7 +122,6 @@ class EbuildMetadataPhase(SubProcess): self._raw_metadata = [] files.ebuild = master_fd self.scheduler.add_reader(files.ebuild, self._output_handler) -self._registered = True retval = portage.doebuild( ebuild_path, @@ -150,16 +155,6 @@ class EbuildMetadataPhase(SubProcess): self._proc = retval -asyncio.ensure_future( -self._async_start(), loop=self.scheduler -).add_done_callback(self._async_start_done) - -async def _async_start(self): -# Call async check_locale here for bug 923841, but code -# also needs to migrate from _start to here, including -# the self.deallocate_config set_result call. -pass - def _async_start_done(self, future): future.cancelled() or future.result() if self._was_cancelled(): -- 2.41.0
[gentoo-portage-dev] [PATCH 7/9] actions: disable pytest-xdist for spawn start-method (workers crash)
Bug: https://bugs.gentoo.org/924416
Signed-off-by: Zac Medico
---
.github/workflows/ci.yml | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5bffd97206..762999b7cc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -90,5 +90,8 @@ jobs:
- name: Run tests for ${{ matrix.python-version }}
run: |
[[ "${{ matrix.start-method }}" == "spawn" ]] && export
PORTAGE_MULTIPROCESSING_START_METHOD=spawn
- export PYTEST_ADDOPTS="-vv -ra -l -o console_output_style=count -n
$(nproc) --dist=worksteal"
+ # spawn start-method crashes pytest-xdist workers (bug 924416)
+ [[ "${{ matrix.start-method }}" == "spawn" ]] && \
+export PYTEST_ADDOPTS="-vv -ra -l -o console_output_style=count"
|| \
+export PYTEST_ADDOPTS="-vv -ra -l -o console_output_style=count -n
$(nproc) --dist=worksteal"
meson test -C /tmp/build --verbose
--
2.41.0
[gentoo-portage-dev] [PATCH 6/9] async_aux_get: Use EbuildMetadataPhase deallocate_config future
For the portdbapi async_aux_get method, there is not a very
good place to store a config pool, so instead use asyncio.Lock
to manage access to the portdbapi doebuild_settings attribute
when using the main event loop in the main thread. For other
threads, clone a config instance since we do not have a
thread-safe config pool. This cloning is expensive, but since
portage internals do not trigger this case, it suffices for now
(an AssertionError ensures that internals do not trigger it).
For the main event loop running in the main thread, performance
with the asyncio.Lock should not be significantly different to
performance prior to commit c95fc64abf96, since check_locale
results are typically cached and before there was only a single
shared doebuild_settings instance with access serialized via
the EbuildMetadataPhase _start method.
Update async_aux_get callers to use asyncio.ensure_future on
the returned coroutine when needed, since it used to return
a future instead of a coroutine, and sometimes a future is
needed for add_done_callback usage.
In the portdbapi async_fetch_map method, fix a broken reference
to "future" which should have been "aux_get_future", an error
discovered while testing this patch.
Bug: https://bugs.gentoo.org/924319
Fixes: c95fc64abf96 ("EbuildPhase: async_check_locale")
Signed-off-by: Zac Medico
---
lib/portage/_emirrordist/FetchIterator.py | 10 +-
lib/portage/dbapi/porttree.py | 129 +++---
lib/portage/tests/update/test_move_ent.py | 3 +
3 files changed, 97 insertions(+), 45 deletions(-)
diff --git a/lib/portage/_emirrordist/FetchIterator.py
b/lib/portage/_emirrordist/FetchIterator.py
index eaf3e53596..e4fdd092af 100644
--- a/lib/portage/_emirrordist/FetchIterator.py
+++ b/lib/portage/_emirrordist/FetchIterator.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2018 Gentoo Foundation
+# Copyright 2013-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import threading
@@ -14,6 +14,7 @@ from portage.exception import PortageException,
PortageKeyError
from portage.package.ebuild.fetch import DistfileName
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.TaskScheduler import TaskScheduler
+from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from .FetchTask import FetchTask
from _emerge.CompositeTask import CompositeTask
@@ -276,8 +277,11 @@ def _async_fetch_tasks(config, hash_filter, repo_config,
digests_future, cpv, lo
result.set_result(fetch_tasks)
def future_generator():
-yield config.portdb.async_aux_get(
-cpv, ("RESTRICT",), myrepo=repo_config.name, loop=loop
+yield asyncio.ensure_future(
+config.portdb.async_aux_get(
+cpv, ("RESTRICT",), myrepo=repo_config.name, loop=loop
+),
+loop,
)
yield config.portdb.async_fetch_map(cpv, mytree=repo_config.location,
loop=loop)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 61d431f917..4eebe1183f 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2021 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["close_portdbapi_caches", "FetchlistDict", "portagetree",
"portdbapi"]
@@ -41,7 +41,9 @@ from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
+import contextlib
import os as _os
+import threading
import traceback
import warnings
import errno
@@ -239,6 +241,7 @@ class portdbapi(dbapi):
# this purpose because doebuild makes many changes to the config
# instance that is passed in.
self.doebuild_settings = config(clone=self.settings)
+self._doebuild_settings_lock = asyncio.Lock()
self.depcachedir = os.path.realpath(self.settings.depcachedir)
if os.environ.get("SANDBOX_ON") == "1":
@@ -356,6 +359,17 @@ class portdbapi(dbapi):
self._better_cache = None
self._broken_ebuilds = set()
+def __getstate__(self):
+state = self.__dict__.copy()
+# These attributes are not picklable, so they are automatically
+# regenerated after unpickling.
+state["_doebuild_settings_lock"] = None
+return state
+
+def __setstate__(self, state):
+self.__dict__.update(state)
+self._doebuild_settings_lock = asyncio.Lock()
+
def _set_porttrees(self, porttrees):
"""
Consumers, such as emirrordist, may modify the porttrees attribute in
@@ -669,7 +683,7 @@ class portdbapi(dbapi):
self.async_aux_get(mycpv, mylist,
[gentoo-portage-dev] [PATCH 5/9] asyncio: Wrap asyncio.Lock for python 3.9 compat
Wrap asyncio.Lock for compatibility with python 3.9 where the
deprecated loop parameter is required in order to avoid "got
Future attached to a different loop" errors.
The pordbapi async_aux_get method can use asyncio.Lock to
serialize access to its doebuild_settings attribute in order
to prevent issues like bug 924319.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico
---
lib/portage/util/futures/_asyncio/__init__.py | 17 +
1 file changed, 17 insertions(+)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index 8f1b8e8275..b6481c281e 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -9,6 +9,7 @@ __all__ = (
"CancelledError",
"Future",
"InvalidStateError",
+"Lock",
"TimeoutError",
"get_child_watcher",
"get_event_loop",
@@ -22,6 +23,7 @@ __all__ = (
"wait_for",
)
+import sys
import types
import weakref
@@ -35,6 +37,7 @@ from asyncio import (
FIRST_EXCEPTION,
Future,
InvalidStateError,
+Lock as _Lock,
shield,
TimeoutError,
wait_for,
@@ -159,6 +162,20 @@ def iscoroutinefunction(func):
return False
+class Lock(_Lock):
+"""
+Inject loop parameter for python3.9 or less in order to avoid
+"got Future attached to a different loop" errors.
+"""
+
+def __init__(self, **kwargs):
+if sys.version_info >= (3, 10):
+kwargs.pop("loop", None)
+elif "loop" not in kwargs:
+kwargs["loop"] = _safe_loop()._loop
+super().__init__(**kwargs)
+
+
class Task(Future):
"""
Schedule the execution of a coroutine: wrap it in a future. A task
--
2.41.0
[gentoo-portage-dev] [PATCH 4/9] ResolverPlayground: Use egencache to create manifests
Make the ResolverPlayground _create_ebuild_manifests method
call egencache --jobs, which reliably triggers the KeyError
from bug 924319 for multiple tests:
lib/portage/tests/bin/test_doins.py::DoIns::testDoInsFallback Exception in
callback EbuildMetadataPhase._async_start_done()
handle: )>
Traceback (most recent call last):
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "lib/_emerge/EbuildMetadataPhase.py", line 154, in _async_start_done
future.cancelled() or future.result()
^^^
File "lib/_emerge/EbuildMetadataPhase.py", line 130, in _async_start
retval = portage.doebuild(
^
File "lib/portage/package/ebuild/doebuild.py", line 1030, in doebuild
doebuild_environment(
File "lib/portage/package/ebuild/doebuild.py", line 519, in
doebuild_environment
eapi = mysettings.configdict["pkg"]["EAPI"]
File "lib/portage/util/__init__.py", line 1684, in __getitem__
return UserDict.__getitem__(self, item_key)
File "lib/portage/cache/mappings.py", line 175, in __getitem__
return self.data[key]
~^
KeyError: 'EAPI'
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico
---
lib/portage/tests/dbapi/test_portdb_cache.py | 4 +-
.../tests/resolver/ResolverPlayground.py | 38 ++-
2 files changed, 23 insertions(+), 19 deletions(-)
diff --git a/lib/portage/tests/dbapi/test_portdb_cache.py
b/lib/portage/tests/dbapi/test_portdb_cache.py
index c7c6913b49..c24a4f2098 100644
--- a/lib/portage/tests/dbapi/test_portdb_cache.py
+++ b/lib/portage/tests/dbapi/test_portdb_cache.py
@@ -1,6 +1,7 @@
-# Copyright 2012-2023 Gentoo Authors
+# Copyright 2012-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import shutil
import subprocess
import sys
import textwrap
@@ -63,6 +64,7 @@ class PortdbCacheTestCase(TestCase):
python_cmd = (portage_python, "-b", "-Wd", "-c")
test_commands = (
+(lambda: shutil.rmtree(md5_cache_dir) or True,),
(lambda: not os.path.exists(pms_cache_dir),),
(lambda: not os.path.exists(md5_cache_dir),),
python_cmd
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index 2d26012873..75c86b615c 100644
--- a/lib/portage/tests/resolver/ResolverPlayground.py
+++ b/lib/portage/tests/resolver/ResolverPlayground.py
@@ -3,6 +3,7 @@
import bz2
import fnmatch
+import subprocess
import tempfile
import portage
@@ -18,8 +19,6 @@ from portage.const import (
from portage.process import find_binary
from portage.dep import Atom, _repo_separator
from portage.dbapi.bintree import binarytree
-from portage.package.ebuild.config import config
-from portage.package.ebuild.digestgen import digestgen
from portage._sets import load_default_config
from portage._sets.base import InternalPackageSet
from portage.tests import cnf_path
@@ -323,22 +322,25 @@ class ResolverPlayground:
f.write(misc_content)
def _create_ebuild_manifests(self, ebuilds):
-tmpsettings = config(clone=self.settings)
-tmpsettings["PORTAGE_QUIET"] = "1"
-for cpv in ebuilds:
-a = Atom("=" + cpv, allow_repo=True)
-repo = a.repo
-if repo is None:
-repo = "test_repo"
-
-repo_dir = self._get_repo_dir(repo)
-ebuild_dir = os.path.join(repo_dir, a.cp)
-ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] +
".ebuild")
-
-portdb = self.trees[self.eroot]["porttree"].dbapi
-tmpsettings["O"] = ebuild_dir
-if not digestgen(mysettings=tmpsettings, myportdb=portdb):
-raise AssertionError(f"digest creation failed for
{ebuild_path}")
+for repo_name in self._repositories:
+if repo_name == "DEFAULT":
+continue
+egencache_cmd = [
+"egencache",
+f"--repo={repo_name}",
+"--update",
+"--update-manifests",
+"--sign-manifests=n",
+"--strict-manifests=n",
+
f"--repositories-configuration={self.settings['PORTAGE_REPOSITORIES']}",
+f"--jobs={portage.util.cpuinfo.get_cpu_count()}",
+]
+result = subprocess.run(
+egencache_cmd,
+env=self.settings.environ(),
+)
+if result.returncode != os.EX_OK:
+raise AssertionError(f"command failed: {egencache_cmd}")
def _create_binpkgs(self, binpkgs):
# When using BUILD_ID, there can be multiple instances for the
--
2.41.0
[gentoo-portage-dev] [PATCH 3/9] MetadataRegen: Use EbuildMetadataPhase deallocate_config
For EbuildMetadataPhase consumers like MetadataRegen and
depgraph, store a pool of config instances in a config_pool
list, and return instaces to the list when the deallocate_config
future is done.
Bug: https://bugs.gentoo.org/924319
Fixes: c95fc64abf96 ("EbuildPhase: async_check_locale")
Signed-off-by: Zac Medico
---
lib/_emerge/MetadataRegen.py | 16 ++--
lib/_emerge/depgraph.py | 11 +++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/lib/_emerge/MetadataRegen.py b/lib/_emerge/MetadataRegen.py
index d29722b94c..538a94b450 100644
--- a/lib/_emerge/MetadataRegen.py
+++ b/lib/_emerge/MetadataRegen.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
@@ -44,6 +44,7 @@ class MetadataRegen(AsyncScheduler):
valid_pkgs = self._valid_pkgs
cp_set = self._cp_set
consumer = self._consumer
+config_pool = []
portage.writemsg_stdout("Regenerating cache entries...\n")
for cp in self._cp_iter:
@@ -73,12 +74,23 @@ class MetadataRegen(AsyncScheduler):
consumer(cpv, repo_path, metadata, ebuild_hash,
True)
continue
+if config_pool:
+settings = config_pool.pop()
+else:
+settings = portage.config(clone=portdb.settings)
+
+deallocate_config = self.scheduler.create_future()
+deallocate_config.add_done_callback(
+lambda future: config_pool.append(future.result())
+)
+
yield EbuildMetadataPhase(
cpv=cpv,
ebuild_hash=ebuild_hash,
portdb=portdb,
repo_path=repo_path,
-settings=portdb.doebuild_settings,
+settings=settings,
+deallocate_config=deallocate_config,
write_auxdb=self._write_auxdb,
)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 1674fa289e..70b83ee1f4 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -754,6 +754,7 @@ class depgraph:
def _dynamic_deps_preload(self, fake_vartree):
portdb = fake_vartree._portdb
+config_pool = []
for pkg in fake_vartree.dbapi:
self._spinner_update()
self._dynamic_config._package_tracker.add_installed_pkg(pkg)
@@ -768,12 +769,22 @@ class depgraph:
if metadata is not None:
fake_vartree.dynamic_deps_preload(pkg, metadata)
else:
+if config_pool:
+settings = config_pool.pop()
+else:
+settings = portage.config(clone=portdb.settings)
+
+deallocate_config = portdb._event_loop.create_future()
+deallocate_config.add_done_callback(
+lambda future: config_pool.append(future.result())
+)
proc = EbuildMetadataPhase(
cpv=pkg.cpv,
ebuild_hash=ebuild_hash,
portdb=portdb,
repo_path=repo_path,
settings=portdb.doebuild_settings,
+deallocate_config=deallocate_config,
)
proc.addExitListener(self._dynamic_deps_proc_exit(pkg,
fake_vartree))
yield proc
--
2.41.0
[gentoo-portage-dev] [PATCH 2/9] EbuildMetadataPhase: Add deallocate_config future
Use a deallocate_config future to release self.settings when
it is no longer needed. It's necessary to manage concurrency
since commit c95fc64abf96 because mutation of self.settings
is no longer limited to the EbuildMetadataPhase _start method,
where exclusive access was guaranteed within the main thread.
Add support to the isAlive() method to detect when the
EbuildMetadataPhase has started but the pid is not yet
available (due to async_check_locale usage from commit
c95fc64abf96). This can be used to check if an
EbuildMetadataPhase instance has been successfully started
so that it can be relied upon to set the result of the
deallocate_config future.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico
---
lib/_emerge/EbuildMetadataPhase.py | 36 ++
lib/_emerge/SubProcess.py | 5 -
2 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index f4f685e81c..784712e8cb 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -14,6 +14,7 @@ from portage import os
from portage import _encodings
from portage import _unicode_decode
from portage import _unicode_encode
+from portage.util.futures import asyncio
import fcntl
@@ -33,6 +34,7 @@ class EbuildMetadataPhase(SubProcess):
"portdb",
"repo_path",
"settings",
+"deallocate_config",
"write_auxdb",
) + (
"_eapi",
@@ -127,6 +129,15 @@ class EbuildMetadataPhase(SubProcess):
returnproc=True,
)
settings.pop("PORTAGE_PIPE_FD", None)
+# At this point we can return settings to the caller
+# since we never use it for anything more than an
+# eapi_invalid call after this, and eapi_invalid is
+# insensitive to concurrent modifications.
+if (
+self.deallocate_config is not None
+and not self.deallocate_config.cancelled()
+):
+self.deallocate_config.set_result(settings)
os.close(slave_fd)
null_input.close()
@@ -139,6 +150,31 @@ class EbuildMetadataPhase(SubProcess):
self._proc = retval
+asyncio.ensure_future(
+self._async_start(), loop=self.scheduler
+).add_done_callback(self._async_start_done)
+
+async def _async_start(self):
+# Call async check_locale here for bug 923841, but code
+# also needs to migrate from _start to here, including
+# the self.deallocate_config set_result call.
+pass
+
+def _async_start_done(self, future):
+future.cancelled() or future.result()
+if self._was_cancelled():
+pass
+elif future.cancelled():
+self.cancel()
+self._was_cancelled()
+
+if self.deallocate_config is not None and not
self.deallocate_config.done():
+self.deallocate_config.set_result(self.settings)
+
+if self.returncode is not None:
+self._unregister()
+self.wait()
+
def _output_handler(self):
while True:
buf = self._read_buf(self._files.ebuild)
diff --git a/lib/_emerge/SubProcess.py b/lib/_emerge/SubProcess.py
index 029bbc3f44..057e0adc24 100644
--- a/lib/_emerge/SubProcess.py
+++ b/lib/_emerge/SubProcess.py
@@ -18,9 +18,12 @@ class SubProcess(AbstractPollTask):
# we've sent a kill signal to our subprocess.
_cancel_timeout = 1 # seconds
+def isAlive(self):
+return (self._registered or self.pid is not None) and self.returncode
is None
+
@property
def pid(self):
-return self._proc.pid
+return None if self._proc is None else self._proc.pid
def _poll(self):
# Simply rely on _async_waitpid_cb to set the returncode.
--
2.41.0
[gentoo-portage-dev] [PATCH 1/9] anydbm: Pickle support for multiprocessing spawn
The egencache usage in ResolverPlayground that was used to trigger
bug 924319 triggered a pickling error for AuxdbTestCase.test_anydbm
with the multiprocessing spawn start method, so fix the anydbm
cache module to omit the unpicklable database object from pickled
state, and regenerate it after unpickling.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico
---
lib/portage/cache/anydbm.py | 17 -
lib/portage/tests/dbapi/test_auxdb.py | 4 +---
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/lib/portage/cache/anydbm.py b/lib/portage/cache/anydbm.py
index 94a270a483..ad7042ae41 100644
--- a/lib/portage/cache/anydbm.py
+++ b/lib/portage/cache/anydbm.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2020 Gentoo Authors
+# Copyright 2005-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# Author(s): Brian Harring (ferri...@gentoo.org)
@@ -67,6 +67,21 @@ class database(fs_template.FsBased):
raise cache_errors.InitializationError(self.__class__, e)
self._ensure_access(self._db_path)
+def __getstate__(self):
+state = self.__dict__.copy()
+# These attributes are not picklable, so they are automatically
+# regenerated after unpickling.
+state["_database__db"] = None
+return state
+
+def __setstate__(self, state):
+self.__dict__.update(state)
+mode = "w"
+if dbm.whichdb(self._db_path) in ("dbm.gnu", "gdbm"):
+# Allow multiple concurrent writers (see bug #53607).
+mode += "u"
+self.__db = dbm.open(self._db_path, mode, self._perms)
+
def iteritems(self):
# dbm doesn't implement items()
for k in self.__db.keys():
diff --git a/lib/portage/tests/dbapi/test_auxdb.py
b/lib/portage/tests/dbapi/test_auxdb.py
index 0de0123a5f..aac6ce361c 100644
--- a/lib/portage/tests/dbapi/test_auxdb.py
+++ b/lib/portage/tests/dbapi/test_auxdb.py
@@ -16,9 +16,7 @@ class AuxdbTestCase(TestCase):
from portage.cache.anydbm import database
except ImportError:
self.skipTest("dbm import failed")
-self._test_mod(
-"portage.cache.anydbm.database", multiproc=False, picklable=False
-)
+self._test_mod("portage.cache.anydbm.database", multiproc=False,
picklable=True)
def test_flat_hash_md5(self):
self._test_mod("portage.cache.flat_hash.md5_database")
--
2.41.0
[gentoo-portage-dev] [PATCH 0/9] EbuildPhase/EbuildMetadataPhase: async check_locale
Change config.environ() check_locale calls to async_check_locale calls in the EbuildPhase/EbuildMetadataPhase _async_start method in order to eliminate synchronous waiting for child processes in the main event loop thread. Note that this series of changes causes access to the portdbapi doebuild_settings attribute to no longer be serialized via the EbuildMetadataPhase _start_method. As a result, exclusive access to config instances needs to be guaranteed in some other way to avoid triggering problems (see bug 924319), such as by maintaining a config pool or by serializing access via an asyncio.Lock instance. This series can also be reviewed at https://github.com/gentoo/portage/pull/1267. Bug: https://bugs.gentoo.org/923841 Bug: https://bugs.gentoo.org/924319 Signed-off-by: Zac Medico Zac Medico (9): anydbm: Pickle support for multiprocessing spawn EbuildMetadataPhase: Add deallocate_config future MetadataRegen: Use EbuildMetadataPhase deallocate_config ResolverPlayground: Use egencache to create manifests asyncio: Wrap asyncio.Lock for python 3.9 compat async_aux_get: Use EbuildMetadataPhase deallocate_config future actions: disable pytest-xdist for spawn start-method (workers crash) EbuildMetadataPhase: Migrate to _async_start EbuildPhase: async_check_locale .github/workflows/ci.yml | 5 +- lib/_emerge/EbuildMetadataPhase.py| 37 - lib/_emerge/EbuildPhase.py| 28 +++- lib/_emerge/MetadataRegen.py | 16 ++- lib/_emerge/SubProcess.py | 5 +- lib/_emerge/depgraph.py | 11 ++ lib/portage/_emirrordist/FetchIterator.py | 10 +- lib/portage/cache/anydbm.py | 17 ++- lib/portage/dbapi/porttree.py | 129 -- lib/portage/package/ebuild/config.py | 26 ++-- lib/portage/tests/dbapi/test_auxdb.py | 4 +- lib/portage/tests/dbapi/test_portdb_cache.py | 4 +- .../tests/resolver/ResolverPlayground.py | 38 +++--- lib/portage/tests/update/test_move_ent.py | 3 + lib/portage/util/futures/_asyncio/__init__.py | 26 lib/portage/util/locale.py| 28 ++-- 16 files changed, 289 insertions(+), 98 deletions(-) -- 2.41.0
[gentoo-portage-dev] [PATCH v2] process.spawn: Add returnproc parameter
In order to migrate away from unsafe os.fork() usage in threaded
processes (https://github.com/python/cpython/issues/84559), add a
returnproc parameter that is similar to returnpid, which causes
spawn to return a single Process object instead of a list of pids.
The Process API is a subset of asyncio.subprocess.Process. The
returnproc parameter conflicts with the logfile parameter, since
the caller is expected to use the fd_pipes parameter to implement
logging (this was also true for the returnpid parameter).
In the future, spawn will return objects of a different type but
with a compatible interface to Process, in order to encapsulate
implementation-dependent objects like multiprocessing.Process which
are designed to manage the process lifecycle and need to persist
until it exits.
Trigger a UserWarning when the returnpid parameter is used, in
order to encourage migration to returnproc (do not use
DeprecationWarning since it is hidden by default). This warning
will be temporarily suppressed for portage internals, until they
finish migrating to returnproc. There are probably very few if
any external consumers of spawn with the returnpid parameter,
so it seems safe to move quickly with this deprecation.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico
---
[PATCH v2] Make returnproc return a singlar Process (for
comparison, returnpid never returned a list container more than
a single pid). Also, returnproc explicitly conflicts with the
logfile parameter, since callers are expected to use the fd_pipes
parameter to implement logging (this was also true for returnpid).
lib/portage/process.py| 84 +++
lib/portage/tests/process/meson.build | 1 +
.../tests/process/test_spawn_returnproc.py| 39 +
3 files changed, 124 insertions(+)
create mode 100644 lib/portage/tests/process/test_spawn_returnproc.py
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 71750a715f..6ec52efc4a 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -15,6 +15,7 @@ import subprocess
import sys
import traceback
import os as _os
+import warnings
from dataclasses import dataclass
from functools import lru_cache
@@ -27,6 +28,7 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util:dump_traceback,writemsg,writemsg_level",
)
@@ -277,12 +279,78 @@ def calc_env_stats(env) -> EnvStats:
env_too_large_warnings = 0
+class Process:
+"""
+An object that wraps OS processes created by spawn.
+In the future, spawn will return objects of a different type
+but with a compatible interface to this class, in order
+to encapsulate implementation-dependent objects like
+multiprocessing.Process which are designed to manage
+the process lifecycle and need to persist until it exits.
+"""
+
+def __init__(self, pid):
+self.pid = pid
+self.returncode = None
+self._exit_waiters = []
+
+def __repr__(self):
+return f"<{self.__class__.__name__} {self.pid}>"
+
+async def wait(self):
+"""
+Wait for the child process to terminate.
+
+Set and return the returncode attribute.
+"""
+if self.returncode is not None:
+return self.returncode
+
+loop = global_event_loop()
+if not self._exit_waiters:
+loop._asyncio_child_watcher.add_child_handler(self.pid,
self._child_handler)
+waiter = loop.create_future()
+self._exit_waiters.append(waiter)
+return await waiter
+
+def _child_handler(self, pid, returncode):
+if pid != self.pid:
+raise AssertionError(f"expected pid {self.pid}, got {pid}")
+self.returncode = returncode
+
+for waiter in self._exit_waiters:
+if not waiter.cancelled():
+waiter.set_result(returncode)
+self._exit_waiters = None
+
+def send_signal(self, sig):
+"""Send a signal to the process."""
+if self.returncode is not None:
+# Skip signalling a process that we know has already died.
+return
+
+try:
+os.kill(self.pid, sig)
+except ProcessLookupError:
+# Suppress the race condition error; bpo-40550.
+pass
+
+def terminate(self):
+"""Terminate the process with SIGTERM"""
+self.send_signal(signal.SIGTERM)
+
+def kill(self):
+"""Kill the process with SIGKILL"""
+self.send_signal(signal.SIGKILL)
+
+
def spawn(
mycommand,
env=None,
opt_name=None,
fd_pipes=None,
returnpid=False,
+returnproc=False,
uid=None,
gid=None,
groups
[gentoo-portage-dev] [PATCH] process.spawn: Add returnproc parameter
In order to migrate away from unsafe os.fork() usage in threaded
processes (https://github.com/python/cpython/issues/84559), add a
returnproc parameter that is similar to returnpid, which causes
spawn to return Process objects instead of pids. The Process
API is a subset of asyncio.subprocess.Process.
In the future, spawn will return objects of a different type but
with a compatible interface to Process, in order to encapsulate
implementation-dependent objects like multiprocessing.Process which
are designed to manage the process lifecycle and need to persist
until it exits.
Trigger a UserWarning when the returnpid parameter is used, in
order to encourage migration to returnproc (do not use
DeprecationWarning since it is hidden by default). This warning
will be temporarily suppressed for portage internals, until they
finish migrating to returnproc. There are probably very few if
any external consumers of spawn with the returnpid parameter,
so it seems safe to move quickly with this deprecation.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico
---
lib/portage/process.py| 79 +++
lib/portage/tests/process/meson.build | 1 +
.../tests/process/test_spawn_returnproc.py| 39 +
3 files changed, 119 insertions(+)
create mode 100644 lib/portage/tests/process/test_spawn_returnproc.py
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 71750a715f..74953b3e22 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -15,6 +15,7 @@ import subprocess
import sys
import traceback
import os as _os
+import warnings
from dataclasses import dataclass
from functools import lru_cache
@@ -27,6 +28,7 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util:dump_traceback,writemsg,writemsg_level",
)
@@ -277,12 +279,78 @@ def calc_env_stats(env) -> EnvStats:
env_too_large_warnings = 0
+class Process:
+"""
+An object that wraps OS processes created by spawn.
+In the future, spawn will return objects of a different type
+but with a compatible interface to this class, in order
+to encapsulate implementation-dependent objects like
+multiprocessing.Process which are designed to manage
+the process lifecycle and need to persist until it exits.
+"""
+
+def __init__(self, pid):
+self.pid = pid
+self.returncode = None
+self._exit_waiters = []
+
+def __repr__(self):
+return f"<{self.__class__.__name__} {self.pid}>"
+
+async def wait(self):
+"""
+Wait for the child process to terminate.
+
+Set and return the returncode attribute.
+"""
+if self.returncode is not None:
+return self.returncode
+
+loop = global_event_loop()
+if not self._exit_waiters:
+loop._asyncio_child_watcher.add_child_handler(self.pid,
self._child_handler)
+waiter = loop.create_future()
+self._exit_waiters.append(waiter)
+return await waiter
+
+def _child_handler(self, pid, returncode):
+if pid != self.pid:
+raise AssertionError(f"expected pid {self.pid}, got {pid}")
+self.returncode = returncode
+
+for waiter in self._exit_waiters:
+if not waiter.cancelled():
+waiter.set_result(returncode)
+self._exit_waiters = None
+
+def send_signal(self, sig):
+"""Send a signal to the process."""
+if self.returncode is not None:
+# Skip signalling a process that we know has already died.
+return
+
+try:
+os.kill(self.pid, sig)
+except ProcessLookupError:
+# Suppress the race condition error; bpo-40550.
+pass
+
+def terminate(self):
+"""Terminate the process with SIGTERM"""
+self.send_signal(signal.SIGTERM)
+
+def kill(self):
+"""Kill the process with SIGKILL"""
+self.send_signal(signal.SIGKILL)
+
+
def spawn(
mycommand,
env=None,
opt_name=None,
fd_pipes=None,
returnpid=False,
+returnproc=False,
uid=None,
gid=None,
groups=None,
@@ -315,6 +383,9 @@ def spawn(
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will
clean them.
@type returnpid: Boolean
+@param returnproc: Return the Process objects for a successful spawn.
+NOTE: This requires the caller clean up all the PIDs, otherwise spawn will
clean them.
+@type returnproc: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
Re: [gentoo-dev] sys-devel/autoconf wrong dependency?
On 1/31/24 04:56, Andreas Fink wrote:
With the move of sys-devel/autoconf to dev-build/autoconf the ebuild
has some inconsistency, namely in the RDEPEND section, it is saying:
RDEPEND="
${BDEPEND}
>=dev-build/autoconf-wrapper-20231224
sys-devel/gnuconfig
!~sys-devel/${P}:2.5
"
This should probably be `!~dev-build/${P}:2.5`. It shows up in the
following as a problem:
emerge -auvDN --changed-deps=y --with-bdeps=y @world # --> merges
autoconf-2.72-r1
Now:
emaint --fix all # --> fixes dependency from sys-devel/autoconf to
dev-build/autoconf
and again a full system update:
emerge -auvDN --changed-deps=y --with-bdeps=y @world # --> merges
autoconf-2.72-r1
This would want to again install autoconf, due to the changed-deps flag.
I am quite sure that this is not the intended behaviour, or is it?
Best
Andreas
Yeah, that's not intended. Reported as https://bugs.gentoo.org/923424.
--
Thanks,
Zac
OpenPGP_signature.asc
Description: OpenPGP digital signature
[gentoo-dev] Last rites: net-misc/drive
commit ba6f1c6fd9b9434bd2c07cf7233ee38cb6ab430a Author: Brian Harring AuthorDate: 2023-11-09 20:51:11 -0800 Commit: Zac Medico CommitDate: 2023-11-09 21:59:23 -0800 net-misc/drive: treeclean Dead upstream and fully broken since 2023-02 due to google auth changes. Closes: https://bugs.gentoo.org/658028 Closes: https://bugs.gentoo.org/903862 Signed-off-by: Brian Harring Closes: https://github.com/gentoo/gentoo/pull/33751 Signed-off-by: Zac Medico OpenPGP_signature Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH 2/2] Add caching to _slot_operator_check_reverse_dependencies
On 11/24/22 19:36, Pin-yen Lin wrote: Add lru_cache to speed up the running time of "Calculating dependencies". In a ChromeOS use case, this patch decreases the running time from 311s to 197s with almost no memory usage increase. Signed-off-by: Pin-yen Lin --- lib/_emerge/depgraph.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index ce6cabcc1..9649bb2a8 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -2240,6 +2240,7 @@ class depgraph: return None +@functools.lru_cache(maxsize=100) def _slot_operator_check_reverse_dependencies( self, existing_pkg, candidate_pkg, replacement_parent=None ): Merged. Thank you! https://gitweb.gentoo.org/proj/portage.git/commit/?id=0c42cc962e1926ecbdc83d903a2804f9e037f2a9 https://gitweb.gentoo.org/proj/portage.git/commit/?id=839ab46be1777e5886da28b98b53a462b992c5bf -- Thanks, Zac
Re: [gentoo-portage-dev] usage of /bin/bash in shebangs
On 7/24/22 23:17, Fabian Groffen wrote: On 24-07-2022 13:58:31 -0700, Zac Medico wrote: On 7/24/22 12:29, Fabian Groffen wrote: Hi, Quick question, I noticed that portage uses /bin/bash hardcoded in shebang of scripts, while it uses /usr/bin/env python for python executable files. Is there anything against using /usr/bin/env bash for shell scripts? Changing this would help for Prefix. I can't think of any reason not to do this. Do you want to see a patch, or is it OK to push this myself? It's OK to just push this yourself. Thanks! -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
Re: [gentoo-portage-dev] usage of /bin/bash in shebangs
On 7/24/22 12:29, Fabian Groffen wrote: Hi, Quick question, I noticed that portage uses /bin/bash hardcoded in shebang of scripts, while it uses /usr/bin/env python for python executable files. Is there anything against using /usr/bin/env bash for shell scripts? Changing this would help for Prefix. I can't think of any reason not to do this. -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
Re: [gentoo-portage-dev] Normaliser function for distfiles
On 5/16/22 10:37, Markus Walter wrote: Hello all, is it possible to do the following: after fetching a distfile portage runs an external normaliser program specified in an ebuild before checking the hash? My use case is the following: I would like to improve the gs-elpa program and provide a precomputed overlay for melpa. However the melpa distfiles are rebuilt everyday and cause checksum failures. However the only thing changing are the timestamps. Hence if a normaliser program could simply set all timestamps to some predefined value (say 1.1.1970) then this problem should vanish. Thanks in advance Markus The only usable hook that we currently have for this is FETCHCOMMAND and RESUMCOMMAND in make.conf. You can replace them with a script that does the normal thing and then sets the timestamp. The default values are found in /usr/share/portage/config/make.globals. -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] MergeProcess: propagate mtimedb["ldpath"] to parent process (bug 836375)
Use an instance of multiprocessing.Pipe to propagate mtimedb["ldpath"]
from the MergeProcess child process to the parent process. This fixes
env_update calls to avoid unnecessary regeneration of ld.so.cache in
cases where mtimedb["ldpath"] has not changed since the last call to
env_update.
Bug: https://bugs.gentoo.org/836375
---
lib/portage/dbapi/_MergeProcess.py | 18 ++
lib/portage/dbapi/vartree.py | 10 ++
2 files changed, 28 insertions(+)
diff --git a/lib/portage/dbapi/_MergeProcess.py
b/lib/portage/dbapi/_MergeProcess.py
index db3f3b105..667a5bf20 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -2,6 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
import io
+import multiprocessing
import platform
import fcntl
@@ -38,6 +39,7 @@ class MergeProcess(ForkProcess):
"_dblink",
"_elog_keys",
"_locked_vdb",
+"_mtime_reader",
)
def _start(self):
@@ -113,6 +115,15 @@ class MergeProcess(ForkProcess):
self._elog_reader_fd = None
return False
+def _mtime_handler(self):
+try:
+mtimes = self._mtime_reader.recv()
+except EOFError:
+pass
+else:
+self.prev_mtimes.clear()
+self.prev_mtimes.update(mtimes)
+
def _spawn(self, args, fd_pipes, **kwargs):
"""
Extend the superclass _spawn method to perform some pre-fork and
@@ -127,6 +138,11 @@ class MergeProcess(ForkProcess):
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
)
+mtime_reader, mtime_writer = multiprocessing.Pipe(duplex=False)
+fd_pipes[mtime_writer.fileno()] = mtime_writer.fileno()
+self.scheduler.add_reader(mtime_reader.fileno(), self._mtime_handler)
+self._mtime_reader = mtime_reader
+
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
@@ -142,6 +158,7 @@ class MergeProcess(ForkProcess):
vartree=self.vartree,
blockers=blockers,
pipe=elog_writer_fd,
+mtime_pipe=mtime_writer,
)
fd_pipes[elog_writer_fd] = elog_writer_fd
self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler)
@@ -160,6 +177,7 @@ class MergeProcess(ForkProcess):
self._elog_reader_fd = elog_reader_fd
pids = super(MergeProcess, self)._spawn(args, fd_pipes, **kwargs)
os.close(elog_writer_fd)
+mtime_writer.close()
self._buf = ""
self._elog_keys = set()
# Discard messages which will be collected by the subprocess,
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 602913862..a95d60691 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1806,6 +1806,7 @@ class dblink:
blockers=None,
scheduler=None,
pipe=None,
+mtime_pipe=None,
):
"""
Creates a DBlink object for a given CPV.
@@ -1862,6 +1863,7 @@ class dblink:
self._device_path_map = {}
self._hardlink_merge_map = {}
self._hash_key = (self._eroot, self.mycpv)
+self._mtime_pipe = mtime_pipe
self._protect_obj = None
self._pipe = pipe
self._postinst_failure = False
@@ -2618,6 +2620,7 @@ class dblink:
writemsg_level=self._display_merge,
vardbapi=self.vartree.dbapi,
)
+self._send_mtimes(ldpath_mtimes)
unmerge_with_replacement = preserve_paths is not None
if not unmerge_with_replacement:
@@ -4243,6 +4246,12 @@ class dblink:
def _emerge_log(self, msg):
emergelog(False, msg)
+def _send_mtimes(self, mtimes):
+if self._mtime_pipe is None:
+return
+
+self._mtime_pipe.send(mtimes)
+
def treewalk(
self,
srcroot,
@@ -5274,6 +5283,7 @@ class dblink:
writemsg_level=self._display_merge,
vardbapi=self.vartree.dbapi,
)
+self._send_mtimes(prev_mtimes)
# For gcc upgrades, preserved libs have to be removed after the
# the library path has been updated.
--
2.35.1
Re: [gentoo-portage-dev] The build system (and install layout) of Portage
On 3/17/22 10:22, Michał Górny wrote: Hi, everyone. You've probably had the opportunity to hear that a lot has changed in Python packaging since Portage's setup.py was written in 2014. There were some minor changes to keep it working since but it's time to reconsider. Long story short, distutils is strongly deprecated, setuptools deprecated most of the customizations (and we're relying heavily on customizations), PEP 517 doesn't cover our use cases exactly... and it's quite likely that sooner or later our build system will fall apart. On top of that, setuptools is going through a stage of "let's vendor a new dependency every week", so it doesn't look like a feasible long-term solution. A large part of the problem is that Portage is heavily relying on non-Pythonic idioms for installing stuff. I wonder if we can rely less on the deprecated customizations somehow. For example, the venv_data_files function in setup.py succeeds in installing a bunch of files in custom locations, and maybe we can rely more on that approach (use it not only for venv installations). -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] Revert "dep_zapdeps: avoid new slots when appropriate (bug 828136)"
Revert the change from bug 828136, since it prevents solving
of some blockers unless --update and --deep are specified as
reported in bug 833014.
Bug: https://bugs.gentoo.org/833014
Reverts: a7289ac0eaaa0d435bf6d9bfb2724a6b39adcbee
Signed-off-by: Zac Medico
---
lib/portage/dep/dep_check.py | 6 +-
.../tests/resolver/test_installkernel.py | 20 +--
.../resolver/test_unecessary_slot_upgrade.py | 11 --
3 files changed, 2 insertions(+), 35 deletions(-)
diff --git a/lib/portage/dep/dep_check.py b/lib/portage/dep/dep_check.py
index 8ca4c0b9d..9fccda08b 100644
--- a/lib/portage/dep/dep_check.py
+++ b/lib/portage/dep/dep_check.py
@@ -376,7 +376,6 @@ def dep_zapdeps(
# c) contains masked installed packages
# d) is the first item
-no_new_slots = []
preferred_in_graph = []
preferred_installed = preferred_in_graph
preferred_any_slot = preferred_in_graph
@@ -392,7 +391,6 @@ def dep_zapdeps(
# unsat_use_* must come after preferred_non_installed
# for correct ordering in cases like || ( foo[a] foo[b] ).
choice_bins = (
-no_new_slots,
preferred_in_graph,
preferred_non_installed,
unsat_use_in_graph,
@@ -691,9 +689,7 @@ def dep_zapdeps(
other.append(this_choice)
else:
if all_use_satisfied:
-if new_slot_count == 0 and not want_update:
-no_new_slots.append(this_choice)
-elif all_in_graph:
+if all_in_graph:
preferred_in_graph.append(this_choice)
elif all_installed:
if all_installed_slots:
diff --git a/lib/portage/tests/resolver/test_installkernel.py
b/lib/portage/tests/resolver/test_installkernel.py
index b73bbe5bb..5909b53aa 100644
--- a/lib/portage/tests/resolver/test_installkernel.py
+++ b/lib/portage/tests/resolver/test_installkernel.py
@@ -58,25 +58,8 @@ class InstallKernelTestCase(TestCase):
),
],
),
-# Demonstrate bug 833014, where the calculation fails unless
+# Test bug 833014, where the calculation failed unless
# --update and --deep are specified.
-ResolverPlaygroundTestCase(
-[
-"sys-kernel/installkernel-systemd-boot",
-"sys-kernel/gentoo-kernel-bin",
-],
-ambiguous_merge_order=True,
-success=False,
-mergelist=[
-"sys-kernel/installkernel-systemd-boot-1",
-"sys-kernel/gentoo-kernel-bin-5.15.23",
-"virtual/dist-kernel-5.15.23",
-(
-"!sys-kernel/installkernel-gentoo",
-"!sys-kernel/installkernel-systemd-boot",
-),
-],
-),
ResolverPlaygroundTestCase(
[
"sys-kernel/installkernel-systemd-boot",
@@ -84,7 +67,6 @@ class InstallKernelTestCase(TestCase):
],
ambiguous_merge_order=True,
success=True,
-options={"--deep": True, "--update": True},
mergelist=[
"virtual/dist-kernel-5.15.23",
"sys-kernel/installkernel-systemd-boot-1",
diff --git a/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
b/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
index f8b8b346a..a89ebdb67 100644
--- a/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
+++ b/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
@@ -26,13 +26,6 @@ class UnnecessarySlotrUpgradeTestCase(TestCase):
test_cases = (
# Test bug 828136, where an unnecessary python slot upgrade
# was triggered.
-ResolverPlaygroundTestCase(
-[
-"app-misc/a",
-],
-success=True,
-mergelist=("app-misc/a-1",),
-),
ResolverPlaygroundTestCase(
[
"app-misc/a",
@@ -42,10 +35,6 @@ class UnnecessarySlotrUpgradeTestCase(TestCase):
"dev-lang/python-3.10",
"app-misc/a-1",
),
-options={
-"--deep": True,
-"--update": True,
-},
),
)
--
2.34.1
Re: [gentoo-portage-dev] [PATCH 4/4] portage.eapi: use functools @lru_cache decorator instead of custom implementation
On 2/26/22 10:04, Zac Medico wrote: On 2/23/22 20:14, Matt Turner wrote: From: "Wolfgang E. Sanyer" Reviewed-by: Matt Turner Signed-off-by: Wolfgang E. Sanyer --- Â lib/portage/eapi.py | 155 Â 1 file changed, 72 insertions(+), 83 deletions(-) diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py index 56e64620a..efcc6c2a0 100644 --- a/lib/portage/eapi.py +++ b/lib/portage/eapi.py @@ -2,12 +2,10 @@ Â # Distributed under the terms of the GNU General Public License v2 Â import collections -import operator -import types - -from portage import eapi_is_supported +from functools import lru_cache +@lru_cache(None) Â def eapi_has_iuse_defaults(eapi): Â if eapi is None: Â return True @@ -15,6 +13,7 @@ def eapi_has_iuse_defaults(eapi): Â return eapi != "0" I think this patch misses the point of the original caching mechanism. It doesn't make sense to cache results of the individual eapi_* functions if they no longer contribute to the _eapi_attrs cache. To clarify, the only reason that the eapi_* functions were cached was so that they would trigger population of the _eapi_attrs cache. In the absence of this_eapi_attrs cache population feature, I doubt that it's very useful to put the lru_cache on the indivdual eapi_* functions. -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH 4/4] portage.eapi: use functools @lru_cache decorator instead of custom implementation
On 2/23/22 20:14, Matt Turner wrote: From: "Wolfgang E. Sanyer" Reviewed-by: Matt Turner Signed-off-by: Wolfgang E. Sanyer --- lib/portage/eapi.py | 155 1 file changed, 72 insertions(+), 83 deletions(-) diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py index 56e64620a..efcc6c2a0 100644 --- a/lib/portage/eapi.py +++ b/lib/portage/eapi.py @@ -2,12 +2,10 @@ # Distributed under the terms of the GNU General Public License v2 import collections -import operator -import types - -from portage import eapi_is_supported +from functools import lru_cache +@lru_cache(None) def eapi_has_iuse_defaults(eapi): if eapi is None: return True @@ -15,6 +13,7 @@ def eapi_has_iuse_defaults(eapi): return eapi != "0" I think this patch misses the point of the original caching mechanism. It doesn't make sense to cache results of the individual eapi_* functions if they no longer contribute to the _eapi_attrs cache. -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] repoman: Remove http compatibility code for metadata DTD
On 1/25/22 08:44, Ulrich Müller wrote:
Commit 3950d76df says: "The http:// compat can be removed once the
Gentoo repository is updated to use https:// everywhere."
Bug: https://bugs.gentoo.org/552720
Signed-off-by: Ulrich Müller
---
repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py
b/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py
index b4e0ee933..0fb97a0df 100644
--- a/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py
@@ -127,7 +127,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
)
else:
doctype_system = _metadata_xml.docinfo.system_url
-if doctype_system.replace("http://;, "https://;) !=
metadata_dtd_uri:
+if doctype_system != metadata_dtd_uri:
if doctype_system is None:
system_problem = "but it is undefined"
else:
Looks good. Please merge.
--
Thanks,
Zac
OpenPGP_signature
Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] file_copy: handle zero bytes copied by copy_file_range (bug 828844)
When copy_file_range copied zero bytes, fall back to sendfile,
so that we don't call copy_file_range in an infinite loop.
Bug: https://bugs.gentoo.org/828844
Tested-by: John Helmert III
Signed-off-by: Zac Medico
---
src/portage_util_file_copy_reflink_linux.c | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/portage_util_file_copy_reflink_linux.c
b/src/portage_util_file_copy_reflink_linux.c
index c6affe57a..b00b57952 100644
--- a/src/portage_util_file_copy_reflink_linux.c
+++ b/src/portage_util_file_copy_reflink_linux.c
@@ -261,13 +261,14 @@ _reflink_linux_file_copy(PyObject *self, PyObject *args)
_out,
len);
-if (copyfunc_ret < 0) {
+if (copyfunc_ret <= 0) {
error = errno;
-if ((errno == EXDEV || errno == ENOSYS || errno ==
EOPNOTSUPP) &&
+if ((errno == EXDEV || errno == ENOSYS || errno ==
EOPNOTSUPP || copyfunc_ret == 0) &&
copyfunc == cfr_wrapper) {
/* Use sendfile instead of copy_file_range for
* cross-device copies, or when the copy_file_range
- * syscall is not available (less than Linux 4.5).
+ * syscall is not available (less than Linux 4.5),
+ * or when copy_file_range copies zero bytes.
*/
error = 0;
copyfunc = sf_wrapper;
--
2.32.0
[gentoo-portage-dev] [PATCH] dep_zapdeps: avoid new slots when appropriate (bug 828136)
Place choices that do not pull in new slots into a preferred
choice bin, so that they will not be mixed with choices that
contain unnecessary upgrades. This fixes the included test
case so that an unnecessary new python slot is not pulled in.
Bug: https://bugs.gentoo.org/828136
Signed-off-by: Zac Medico
---
lib/portage/dep/dep_check.py | 6 +-
.../resolver/test_unecessary_slot_upgrade.py | 62 +++
2 files changed, 67 insertions(+), 1 deletion(-)
create mode 100644 lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
diff --git a/lib/portage/dep/dep_check.py b/lib/portage/dep/dep_check.py
index 9fccda08b..8ca4c0b9d 100644
--- a/lib/portage/dep/dep_check.py
+++ b/lib/portage/dep/dep_check.py
@@ -376,6 +376,7 @@ def dep_zapdeps(
# c) contains masked installed packages
# d) is the first item
+no_new_slots = []
preferred_in_graph = []
preferred_installed = preferred_in_graph
preferred_any_slot = preferred_in_graph
@@ -391,6 +392,7 @@ def dep_zapdeps(
# unsat_use_* must come after preferred_non_installed
# for correct ordering in cases like || ( foo[a] foo[b] ).
choice_bins = (
+no_new_slots,
preferred_in_graph,
preferred_non_installed,
unsat_use_in_graph,
@@ -689,7 +691,9 @@ def dep_zapdeps(
other.append(this_choice)
else:
if all_use_satisfied:
-if all_in_graph:
+if new_slot_count == 0 and not want_update:
+no_new_slots.append(this_choice)
+elif all_in_graph:
preferred_in_graph.append(this_choice)
elif all_installed:
if all_installed_slots:
diff --git a/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
b/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
new file mode 100644
index 0..f8b8b346a
--- /dev/null
+++ b/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
@@ -0,0 +1,62 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+ResolverPlayground,
+ResolverPlaygroundTestCase,
+)
+
+
+class UnnecessarySlotrUpgradeTestCase(TestCase):
+def testUnnecessarySlotUpgrade(self):
+ebuilds = {
+"app-misc/a-1": {
+"EAPI": "8",
+"RDEPEND": "|| ( dev-lang/python:3.10 dev-lang/python:3.9 ) ||
( dev-lang/python:3.10 dev-lang/python:3.9 )",
+},
+"dev-lang/python-3.9": {"SLOT": "3.9"},
+"dev-lang/python-3.10": {"SLOT": "3.10"},
+}
+
+installed = {
+"dev-lang/python-3.9": {"SLOT": "3.9"},
+}
+
+test_cases = (
+# Test bug 828136, where an unnecessary python slot upgrade
+# was triggered.
+ResolverPlaygroundTestCase(
+[
+"app-misc/a",
+],
+success=True,
+mergelist=("app-misc/a-1",),
+),
+ResolverPlaygroundTestCase(
+[
+"app-misc/a",
+],
+success=True,
+mergelist=(
+"dev-lang/python-3.10",
+"app-misc/a-1",
+),
+options={
+"--deep": True,
+"--update": True,
+},
+),
+)
+
+playground = ResolverPlayground(
+debug=False, ebuilds=ebuilds, installed=installed
+)
+
+try:
+for test_case in test_cases:
+playground.run_TestCase(test_case)
+self.assertEqual(test_case.test_success, True,
test_case.fail_msg)
+finally:
+playground.debug = False
+playground.cleanup()
--
2.32.0
[gentoo-portage-dev] [PATCH] emerge: Default enable soname dependencies (bug 687956)
Default emerge --ignore-soname-deps=n, in order to enable
soname dependencies by default. As always, soname dependencies
remain inapplicable in the absence of the --usepkgonly option
(or --getbinpkgonly). Therefore, this change only affects
commands that specify --usepkgonly or --getbinpkgonly.
Bug: https://bugs.gentoo.org/687956
Signed-off-by: Zac Medico
---
lib/_emerge/create_depgraph_params.py | 2 +-
man/emerge.1 | 7 ---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/lib/_emerge/create_depgraph_params.py
b/lib/_emerge/create_depgraph_params.py
index 11c3e3736..95c4c2035 100644
--- a/lib/_emerge/create_depgraph_params.py
+++ b/lib/_emerge/create_depgraph_params.py
@@ -104,7 +104,7 @@ def create_depgraph_params(myopts, myaction):
if ignore_built_slot_operator_deps is not None:
myparams["ignore_built_slot_operator_deps"] =
ignore_built_slot_operator_deps
-myparams["ignore_soname_deps"] = myopts.get("--ignore-soname-deps", "y")
+myparams["ignore_soname_deps"] = myopts.get("--ignore-soname-deps", "n")
dynamic_deps = myopts.get("--dynamic-deps", "y") != "n" and "--nodeps" not
in myopts
if dynamic_deps:
diff --git a/man/emerge.1 b/man/emerge.1
index 8f6d12925..ff565b46f 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -639,9 +639,10 @@ supported beginning with \fBEAPI 5\fR.
.TP
.BR "\-\-ignore\-soname\-deps < y | n >"
Ignore the soname dependencies of binary and installed packages. This
-option is enabled by default, since soname dependencies are relatively
-new, and the required metadata is not guaranteed to exist for binary and
-installed packages built with older versions of portage. Also, soname
+option may be useful when working with binary or installed packages
+that lack appropriate soname dependency metadata because they were built
+with a package manager that does not support soname dependencies (perhaps
+an older version of portage). Soname
dependencies will be automatically ignored for dependency calculations
that can pull unbuilt ebuilds into the dependency graph, since unbuilt
ebuilds do not have any soname dependency metadata, making it impossible
--
2.32.0
Re: [gentoo-dev] [PATCH] go-module.eclass: Add GO_OPTIONAL flag
On 11/21/21 02:57, Florian Schmaus wrote:
Following the pattern found in other eclasses, add GO_OPTIONAL to the
go-module eclass. This allows to inherit the eclass without pulling
its dependencies. See, e.g., bug #775779 for the motivation.
Signed-off-by: Florian Schmaus
---
eclass/go-module.eclass | 31 ++-
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/eclass/go-module.eclass b/eclass/go-module.eclass
index 3ad8542a28ae..c9eb90ac62ea 100644
--- a/eclass/go-module.eclass
+++ b/eclass/go-module.eclass
@@ -1,4 +1,4 @@
-# Copyright 2019-2020 Gentoo Authors
+# Copyright 2019-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# @ECLASS: go-module.eclass
@@ -55,13 +55,17 @@ if [[ -z ${_GO_MODULE} ]]; then
_GO_MODULE=1
-BDEPEND=">=dev-lang/go-1.12"
+if [[ ! ${GO_OPTIONAL} ]]; then
+ BDEPEND=">=dev-lang/go-1.12"
-# Workaround for pkgcheck false positive: https://github.com/pkgcore/pkgcheck/issues/214
-# MissingUnpackerDep: version ...: missing BDEPEND="app-arch/unzip"
-# Added here rather than to each affected package, so it can be cleaned up just
-# once when pkgcheck is improved.
-BDEPEND+=" app-arch/unzip"
+ # Workaround for pkgcheck false positive:
https://github.com/pkgcore/pkgcheck/issues/214
+ # MissingUnpackerDep: version ...: missing BDEPEND="app-arch/unzip"
+ # Added here rather than to each affected package, so it can be cleaned
up just
+ # once when pkgcheck is improved.
+ BDEPEND+=" app-arch/unzip"
+
+ EXPORT_FUNCTIONS src_unpack
+fi
# Force go to build in module mode.
# In this mode the GOPATH environment variable is ignored.
@@ -83,8 +87,6 @@ QA_FLAGS_IGNORED='.*'
# Go packages should not be stripped with strip(1).
RESTRICT+=" strip"
-EXPORT_FUNCTIONS src_unpack
-
# @ECLASS-VARIABLE: EGO_SUM
# @DESCRIPTION:
# This is an array based on the go.sum content from inside the target package.
@@ -147,6 +149,17 @@ EXPORT_FUNCTIONS src_unpack
# directory structure.
declare -A -g _GOMODULE_GOSUM_REVERSE_MAP
+# @ECLASS-VARIABLE: GO_OPTIONAL
+# @DEFAULT_UNSET
+# @PRE_INHERIT
+# @DESCRIPTION:
+# If set to a non-null value before inherit, then the Go part of the
+# ebuild will be considered optional. No dependencies will be added and
+# no phase functions will be exported.
+#
+# If you enable GO_OPTIONAL, you have to set BDEPEND on >=dev-lang/go-1.12
+# for your package and call go-module_src_unpack manually.
+
# @FUNCTION: go-module_set_globals
# @DESCRIPTION:
# Convert the information in EGO_SUM for other usage in the ebuild.
How about if we also add a GO_DEPEND variable, so that eclasshi
consumers can do something like BDEPEND="go? ( ${GO_DEPEND} )" ?
--
Thanks,
Zac
OpenPGP_signature
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Install example repo.postsync.d script into sharedir
On 11/4/21 02:50, Daniel Cordero wrote:
The sysconfdir is for host specific configuration files, and this
example script makes no host specific change (it is not enabled by
default).
Install the script under portage's sharedir, from where administrators
can copy it into sysconfdir, if needed.
Signed-off-by: Daniel Cordero
---
setup.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/setup.py b/setup.py
index 549fff650..fe0265c64 100755
--- a/setup.py
+++ b/setup.py
@@ -833,12 +833,12 @@ setup(
["$portage_setsdir", ["cnf/sets/portage.conf"]],
["$docdir", ["NEWS", "RELEASE-NOTES"]],
["$portage_base/bin", ["bin/deprecated-path"]],
-["$sysconfdir/portage/repo.postsync.d",
["cnf/repo.postsync.d/example"]],
+["$portage_confdir/repo.postsync.d",
["cnf/repo.postsync.d/example"]],
],
[
("etc", "cnf", ("etc-update.conf", "dispatch-conf.conf")),
("etc/logrotate.d", "cnf/logrotate.d", ("elog-save-summary",)),
-("etc/portage/repo.postsync.d", "cnf/repo.postsync.d",
("example",)),
+("share/portage/config/repo.postsync.d", "cnf/repo.postsync.d",
("example",)),
(
"share/portage/config",
"cnf",
Merged, thanks!
https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7dc908a99e564de6f10174b1489028d939b917f
--
Thanks,
Zac
[gentoo-portage-dev] Re: [PATCH] EbuildIpc.communicate: lockfile PermissionDenied retry
On 11/3/21 20:09, Zac Medico wrote: The lockfile function is expected to raise PermissionDenied if the (root) parent process holds the lock, so retry in this case. Bug: https://bugs.gentoo.org/468990 Signed-off-by: Zac Medico --- bin/ebuild-ipc.py | 11 ++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py index 4999c043a..6eaa658a2 100755 --- a/bin/ebuild-ipc.py +++ b/bin/ebuild-ipc.py @@ -158,7 +158,16 @@ class EbuildIpc: # Make locks quiet since unintended locking messages displayed on # stdout could corrupt the intended output of this program. portage.locks._quiet = True -lock_obj = portage.locks.lockfile(self.ipc_lock_file, unlinkfile=True) +# Acquire lock with PermissionDenied retry for bug #468990. +for _ in range(1000): +try: +lock_obj = portage.locks.lockfile(self.ipc_lock_file, unlinkfile=True) +except portage.exception.PermissionDenied: +time.sleep(0.1) +else: +break +else: +raise portage.exception.PermissionDenied(self.ipc_lock_file) try: return self._communicate(args) Withdrawn in favor of this lockfile permission race fix: https://archives.gentoo.org/gentoo-portage-dev/message/88916062415d9f692091dfb947f1bda2 -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] EbuildIpcDaemon: fix lock permission race
Move ipc files to a .ipc subdirectory, with a setgid bit to
prevent a lockfile group permission race. The lockfile function
uses an appropriate open call with mode argument so that the
lockfile is created atomically with both group ownership and
group write bit.
Bug: https://bugs.gentoo.org/468990
Signed-off-by: Zac Medico
---
bin/ebuild-ipc.py| 6 +++---
bin/phase-functions.sh | 4 ++--
lib/_emerge/AbstractEbuildProcess.py | 4 ++--
lib/_emerge/EbuildIpcDaemon.py | 2 +-
lib/portage/package/ebuild/prepare_build_dirs.py | 6 ++
lib/portage/tests/ebuild/test_doebuild_spawn.py | 1 +
lib/portage/tests/ebuild/test_ipc_daemon.py | 6 +++---
7 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py
index 4999c043a..c24ba6f58 100755
--- a/bin/ebuild-ipc.py
+++ b/bin/ebuild-ipc.py
@@ -138,9 +138,9 @@ class EbuildIpc:
def __init__(self):
self.fifo_dir = os.environ["PORTAGE_BUILDDIR"]
-self.ipc_in_fifo = os.path.join(self.fifo_dir, ".ipc_in")
-self.ipc_out_fifo = os.path.join(self.fifo_dir, ".ipc_out")
-self.ipc_lock_file = os.path.join(self.fifo_dir, ".ipc_lock")
+self.ipc_in_fifo = os.path.join(self.fifo_dir, ".ipc", "in")
+self.ipc_out_fifo = os.path.join(self.fifo_dir, ".ipc", "out")
+self.ipc_lock_file = os.path.join(self.fifo_dir, ".ipc", "lock")
def _daemon_is_alive(self):
try:
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index d3221993d..5eb031805 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -291,10 +291,10 @@ __dyn_clean() {
rm -f
"$PORTAGE_BUILDDIR"/.{ebuild_changed,logid,pretended,setuped,unpacked,prepared}
\
"$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged,instprepped} \
"$PORTAGE_BUILDDIR"/.die_hooks \
- "$PORTAGE_BUILDDIR"/.ipc_{in,out,lock} \
"$PORTAGE_BUILDDIR"/.exit_status
- rm -rf "${PORTAGE_BUILDDIR}/build-info"
+ rm -rf "${PORTAGE_BUILDDIR}/build-info" \
+ "${PORTAGE_BUILDDIR}/.ipc"
rm -rf "${WORKDIR}"
rm -f "${PORTAGE_BUILDDIR}/files"
fi
diff --git a/lib/_emerge/AbstractEbuildProcess.py
b/lib/_emerge/AbstractEbuildProcess.py
index 1b4e7759f..6d89d40f0 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -249,8 +249,8 @@ class AbstractEbuildProcess(SpawnProcess):
def _init_ipc_fifos(self):
-input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in")
-output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".ipc_out")
+input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc",
"in")
+output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc",
"out")
for p in (input_fifo, output_fifo):
diff --git a/lib/_emerge/EbuildIpcDaemon.py b/lib/_emerge/EbuildIpcDaemon.py
index ee6fd7658..78594ff0a 100644
--- a/lib/_emerge/EbuildIpcDaemon.py
+++ b/lib/_emerge/EbuildIpcDaemon.py
@@ -81,7 +81,7 @@ class EbuildIpcDaemon(FifoIpcDaemon):
# write something to the pipe just before we close it, and in that
# case the write will be lost. Therefore, try for a non-blocking
# lock, and only re-open the pipe if the lock is acquired.
-lock_filename = os.path.join(os.path.dirname(self.input_fifo),
".ipc_lock")
+lock_filename = os.path.join(os.path.dirname(self.input_fifo),
"lock")
try:
lock_obj = lockfile(lock_filename, unlinkfile=True,
flags=os.O_NONBLOCK)
except TryAgain:
diff --git a/lib/portage/package/ebuild/prepare_build_dirs.py
b/lib/portage/package/ebuild/prepare_build_dirs.py
index 659198905..9d2474fd8 100644
--- a/lib/portage/package/ebuild/prepare_build_dirs.py
+++ b/lib/portage/package/ebuild/prepare_build_dirs.py
@@ -102,6 +102,12 @@ def prepare_build_dirs(myroot=None, settings=None,
cleanup=False):
apply_secpass_permissions(
mysettings[dir_key], uid=portage_uid, gid=portage_gid
)
+# The setgid bit prevents a lockfile group permission race for bug
#468990.
+ensure_dirs(
+os.path.join(mysettings["PORTAGE_BUILDDIR"], ".ipc"),
+gid=portage_gid,
+mode=0o2770,
+)
except PermissionDenied as e:
writemsg(_("Permission Denied: %s\n") %
[gentoo-portage-dev] [PATCH] fetch: enable resume for digestgen case
Enable resume for the digestgen case (no digests available), when
the temporary file exceeds PORTAGE_FETCH_RESUME_MIN_SIZE. This
fixes a case which caused the ebuild digest command to skip the
download and fail with a message like this:
!!! File b'/var/cache/distfiles/foo.tar.xz' doesn't exist, can't update Manifest
Bug: https://bugs.gentoo.org/821571
Signed-off-by: Zac Medico
---
lib/portage/package/ebuild/fetch.py | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py
b/lib/portage/package/ebuild/fetch.py
index 8c64362c2..2d3625800 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -1485,10 +1485,11 @@ def fetch(
except EnvironmentError:
pass
elif not orig_digests:
-# We don't have a digest, but the file exists. We must
-# assume that it is fully downloaded.
+# We don't have a digest, and the temporary file
exists.
if not force:
-continue
+# Try to resume this download when full
+# download has not been explicitly forced.
+fetched = 1
else:
if (
mydigests[myfile].get("size") is not None
--
2.32.0
Re: [gentoo-portage-dev] [PATCH] bin/estrip: avoid copying directories in FEATURES=installsources
On 7/17/21 12:59 PM, Sergei Trofimovich wrote:
> Initially problem is noticed on gcc-11 as a full ${WORKDIR} syncing
> into /usr/src/debug. It happens because `debug.sources` sometimes
> contains directory. For example on bash-5 it has:
>
> $ grep -zv '/<[^/>]*>$' debug.sources | LANG=C sort -z -u | sed -e
> 's/\x00/\n/g'
> bash-5.0/
> bash-5.0/alias.c
> ...
>
> This causes syncing object files, config.log, final binaries
> and other unexpected data. The change avoids syncking paths
> that end with '/'.
>
> Signed-off-by: Sergei Trofimovich
> ---
> bin/estrip | 3 +++
> 1 file changed, 3 insertions(+)
>
> diff --git a/bin/estrip b/bin/estrip
> index 7ef1ec35c..6cca0d04b 100755
> --- a/bin/estrip
> +++ b/bin/estrip
> @@ -464,7 +464,10 @@ if [[ -s ${tmpdir}/debug.sources ]] && \
> then
> __vecho "installsources: rsyncing source files"
> [[ -d ${D%/}/${prepstrip_sources_dir#/} ]] || mkdir -p
> "${D%/}/${prepstrip_sources_dir#/}"
> + # skip installation of ".../" (system headers? why inner slashes
> are forbidden?)
> + # skip syncing of ".../foo/" (complete directories)
> grep -zv '/<[^/>]*>$' "${tmpdir}"/debug.sources | \
> + grep -zv '/$' | \
> (cd "${WORKDIR}"; LANG=C sort -z -u | \
> rsync -tL0 --chmod=ugo-st,a+r,go-w,Da+x,Fa-x --files-from=-
> "${WORKDIR}/" "${D%/}/${prepstrip_sources_dir#/}/" )
>
>
Looks good. Merged with both grep calls combined via grep -e. Thanks!
https://gitweb.gentoo.org/proj/portage.git/commit/?id=e083c8bf20d8488d329e3dccd643c28429e6fe30
--
Thanks,
Zac
OpenPGP_signature
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] man/make.conf.5: remove mention of zlib USE flag
On 6/28/21 2:56 PM, Thymo van Beers wrote: > Both sys-devel/binutils and sys-devel/gdb are built with system zlib by > default for some time now. This commit removes the mention of USE=zlib to > avoid > confusion. > > Signed-off-by: Thymo van Beers > --- > man/make.conf.5 | 3 +-- > 1 file changed, 1 insertion(+), 2 deletions(-) > > diff --git a/man/make.conf.5 b/man/make.conf.5 > index 1c72109ad..db742fdb5 100644 > --- a/man/make.conf.5 > +++ b/man/make.conf.5 > @@ -378,8 +378,7 @@ redundant on\-the\-fly compression. The resulting file > will be called > .TP > .B compressdebug > Compress the debug sections in the split debug files with zlib to save > -space. Make sure you have built both binutils and gdb with USE=zlib > -support for this to work. See \fBsplitdebug\fR for general split debug > +space. See \fBsplitdebug\fR for general split debug > information (upon which this feature depends). > .TP > .B config\-protect\-if\-modified > Merged thanks! https://gitweb.gentoo.org/proj/portage.git/commit/?id=ee944c4fd76af4f7dffb756e9ed0303cb9606112 -- Thanks, Zac
[gentoo-portage-dev] Re: [PATCH v2] Add @unsatisfied-deps package set (bug 248026)
On 6/18/21 8:29 PM, Zac Medico wrote: > On 6/18/21 8:13 PM, Zac Medico wrote: >> On 6/18/21 6:01 PM, Zac Medico wrote: >>> If emerge --depclean fails to resolve any dependencies, then it will >>> now suggest emerge @unsatisfied-deps as the simplest possible >>> solution, and will also suggest to unmerge @unavailable where >>> appropriate at the end: >>> >>> $ emerge --depclean >>> >>> Calculating dependencies... done! >>> * Dependencies could not be completely resolved due to >>> * the following required packages not being installed: >>> * >>> * virtual/cdrtools pulled in by: >>> * app-cdr/cdrdao-1.2.4 >>> * >>> * Have you forgotten to resolve unsatisfied dependencies prior to >>> * depclean? The simplest possible command for this purpose is as >>> * follows: >>> * >>> * emerge @unsatisfied-deps >> >> It turns out that @unsatisfied-deps is often unsuitable here because it >> pulls in a bunch of @installed packages, when you really want to use >> @world as the source of truth. > > The underlying reason is the same as the reason that we've never used > @installed updates as a substitute for @world updates. It just doesn't > work, because @installed is polluted in comparison to @world. > My plan is it introduce an @unsatisfied-world set which is equivalent to @unsatisfied-deps but filters out any non @world packages. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] Re: [PATCH v2] Add @unsatisfied-deps package set (bug 248026)
On 6/18/21 8:13 PM, Zac Medico wrote: > On 6/18/21 6:01 PM, Zac Medico wrote: >> If emerge --depclean fails to resolve any dependencies, then it will >> now suggest emerge @unsatisfied-deps as the simplest possible >> solution, and will also suggest to unmerge @unavailable where >> appropriate at the end: >> >> $ emerge --depclean >> >> Calculating dependencies... done! >> * Dependencies could not be completely resolved due to >> * the following required packages not being installed: >> * >> * virtual/cdrtools pulled in by: >> * app-cdr/cdrdao-1.2.4 >> * >> * Have you forgotten to resolve unsatisfied dependencies prior to >> * depclean? The simplest possible command for this purpose is as >> * follows: >> * >> * emerge @unsatisfied-deps > > It turns out that @unsatisfied-deps is often unsuitable here because it > pulls in a bunch of @installed packages, when you really want to use > @world as the source of truth. The underlying reason is the same as the reason that we've never used @installed updates as a substitute for @world updates. It just doesn't work, because @installed is polluted in comparison to @world. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] Re: [PATCH v2] Add @unsatisfied-deps package set (bug 248026)
On 6/18/21 6:01 PM, Zac Medico wrote: > If emerge --depclean fails to resolve any dependencies, then it will > now suggest emerge @unsatisfied-deps as the simplest possible > solution, and will also suggest to unmerge @unavailable where > appropriate at the end: > > $ emerge --depclean > > Calculating dependencies... done! > * Dependencies could not be completely resolved due to > * the following required packages not being installed: > * > * virtual/cdrtools pulled in by: > * app-cdr/cdrdao-1.2.4 > * > * Have you forgotten to resolve unsatisfied dependencies prior to > * depclean? The simplest possible command for this purpose is as > * follows: > * > * emerge @unsatisfied-deps It turns out that @unsatisfied-deps is often unsuitable here because it pulls in a bunch of @installed packages, when you really want to use @world as the source of truth. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v2] Add @unsatisfied-deps package set (bug 248026)
If emerge --depclean fails to resolve any dependencies, then it will
now suggest emerge @unsatisfied-deps as the simplest possible
solution, and will also suggest to unmerge @unavailable where
appropriate at the end:
$ emerge --depclean
Calculating dependencies... done!
* Dependencies could not be completely resolved due to
* the following required packages not being installed:
*
* virtual/cdrtools pulled in by:
* app-cdr/cdrdao-1.2.4
*
* Have you forgotten to resolve unsatisfied dependencies prior to
* depclean? The simplest possible command for this purpose is as
* follows:
*
* emerge @unsatisfied-deps
*
* The most comprehensive possible update command is this:
*
* emerge --update --newuse --deep --with-bdeps=y @world
*
* Note that the --with-bdeps=y option is not required in many
* situations. Refer to the emerge manual page (run `man emerge`)
* for more information about --with-bdeps.
*
* Also, note that it may be necessary to manually uninstall
* packages that no longer exist in the repository, since it may not
* be possible to satisfy their dependencies. The simplest possible
* command for this purpose is as follows, but be careful to examine
* the resulting package list carefully:
*
* emerge --ask --unmerge @unavailable
*
Bug: https://bugs.gentoo.org/248026
Signed-off-by: Zac Medico
---
[PATCH v2] Update --depclean message to suggest @unsatisfied-deps
and unmerge @unavailable where appropriate.
cnf/sets/portage.conf | 5 +++
doc/config/sets.docbook | 7
lib/_emerge/actions.py| 19 +++--
lib/portage/_sets/__init__.py | 4 ++
lib/portage/_sets/dbapi.py| 73 ++-
5 files changed, 104 insertions(+), 4 deletions(-)
diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf
index c4ad2efca..2bf38e414 100644
--- a/cnf/sets/portage.conf
+++ b/cnf/sets/portage.conf
@@ -115,3 +115,8 @@ class = portage.sets.dbapi.ChangedDepsSet
class = portage.sets.dbapi.VariableSet
variable = INHERITED
includes = golang-base golang-build golang-vcs golang-vcs-snapshot go-module
+
+# Package set which contains all installed packages having one or more
+# unsatisfied runtime dependencies.
+[unsatisfied-deps]
+class = portage.sets.dbapi.UnsatisfiedDepsSet
diff --git a/doc/config/sets.docbook b/doc/config/sets.docbook
index eba98f468..015ec0c05 100644
--- a/doc/config/sets.docbook
+++ b/doc/config/sets.docbook
@@ -610,6 +610,13 @@
+
+ portage.sets.dbapi.UnsatisfiedDepsSet
+
+ Package set which contains all installed packages
+ having one or more unsatisfied runtime dependencies.
+
+
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 1946f49df..ba2592bba 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1000,11 +1000,18 @@ def _calc_depclean(settings, trees, ldpath_mtimes,
msg.append("%s" % (parent,))
msg.append("")
msg.extend(textwrap.wrap(
- "Have you forgotten to do a complete update
prior " + \
- "to depclean? The most comprehensive command
for this " + \
+ "Have you forgotten to resolve unsatisfied
dependencies prior "
+ "to depclean? The simplest possible command for
this "
"purpose is as follows:", 65
))
msg.append("")
+ msg.append(" " + \
+ good("emerge @unsatisfied-deps"))
+ msg.append("")
+ msg.extend(textwrap.wrap(
+ "The most comprehensive possible update command
is this:", 65
+ ))
+ msg.append("")
msg.append(" " + \
good("emerge --update --newuse --deep
--with-bdeps=y @world"))
msg.append("")
@@ -1018,8 +1025,14 @@ def _calc_depclean(settings, trees, ldpath_mtimes,
msg.extend(textwrap.wrap(
"Also, note that it may be necessary to
manually uninstall " + \
"packages that no longer exist in the
repository, since " + \
- "it may not be possible to satisfy their
dependencies.", 65
+ "it may not be possible to satisfy their
dependencies."
+
[gentoo-dev] Re: [PATCH 0/2] go-module.eclass cleanups
On 5/21/21 8:45 AM, William Hubbs wrote: > This is an improvement to my previous patch. It is a patch series now > because there are two separate changes: > > - GOPROXY is exported in go-module_set_globals since it is not needed if > EGO_SUM is not set in the ebuild. > > - go-module_setup_proxy is added. This function sets up the go proxy > directory so that dependencies can be read from it. > > William Hubbs (2): > go-module.eclass: fix GOPROXY export > go-module.eclass: add go-module_setup_proxy function > > eclass/go-module.eclass | 47 +++-- > 1 file changed, 45 insertions(+), 2 deletions(-) > This series looks great to me! Thank you! -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-dev] Re: [PATCH] go-module.eclass: add functions for use in custom src_unpack phase
On 5/20/21 9:03 AM, William Hubbs wrote:
> On Wed, May 19, 2021 at 01:57:38PM -0700, Zac Medico wrote:
>> On 5/19/21 1:45 PM, Zac Medico wrote:
>>>> +# @FUNCTION: go-module_setup_proxy
>>>> +# @DESCRIPTION:
>>>> +# If your ebuild redefines src_unpack and uses EGO_SUM you need to call
>>>> +# this function in src_unpack.
>>>> +# It sets up the go module proxy in the appropriate location and exports
>>>> +# the GOPROXY environment variable so that go calls will be able to
>>>> +# locate the proxy directory.
>>>> +go-module_setup_proxy() {
>>>> + # shellcheck disable=SC2120
>>>> + debug-print-function "${FUNCNAME}" "$@"
>>>> +
>>>> + if [[ ! ${_GO_MODULE_SET_GLOBALS_CALLED} ]]; then
>>>> + die "go-module_set_globals must be called in global scope"
>>>> + fi
>>>> +
>>>> + local goproxy_dir="${T}/go-proxy"
>>>> + mkdir -p "${goproxy_dir}" || die
>>>> +
>>>> + # For each Golang module distfile, look up where it's supposed to go and
>>>> + # symlink it into place.
>>>> + local f
>>>> + local goproxy_mod_dir
>>>> + for f in ${A}; do
>>>> + goproxy_mod_path="${_GOMODULE_GOSUM_REVERSE_MAP["${f}"]}"
>>>> + if [[ -n "${goproxy_mod_path}" ]]; then
>>>> + debug-print-function "Populating go proxy for
>>>> ${goproxy_mod_path}"
>>>> + # Build symlink hierarchy
>>>> + goproxy_mod_dir=$( dirname
>>>> "${goproxy_dir}"/"${goproxy_mod_path}" )
>>>> + mkdir -p "${goproxy_mod_dir}" || die
>>>> + ln -sf "${DISTDIR}"/"${f}"
>>>> "${goproxy_dir}/${goproxy_mod_path}" ||
>>>> + die "Failed to ln"
>>>> + local v=${goproxy_mod_path}
>>>> + v="${v%.mod}"
>>>> + v="${v%.zip}"
>>>> + v="${v//*\/}"
>>>> + _go-module_gosum_synthesize_files "${goproxy_mod_dir}"
>>>> "${v}"
>>>> + fi
>>>> + done
>>>> + export GOPROXY="file://${goproxy_dir}"
>>>> +
>>>> + # Validate the gosum now
>>>> + _go-module_src_unpack_verify_gosum
>>>> +}
>>>> +
>>>> # @FUNCTION: go-module_src_unpack
>>>> # @DESCRIPTION:
>>>> # If EGO_SUM is set, unpack the base tarball(s) and set up the
>>>>
>>>
>>> The go-module_setup_proxy function solves bug 790851 nicely, since
>>> sys-cluster/k3s ebuilds can call that instead of go-module_src_unpack.
>>
>> I do have one criticism of the go-module_setup_proxy, which is that it
>> relies on the side-effect of the GOPROXY export for its operation. We
>> can instead echo the GOPROXY value to stdout and force the caller to
>> export it themselves, and provide a convenience wrapper function which
>> works based on side-effects.
>
> You really only need GOPROXY if you use EGO_SUM in your ebuild, so I
> could probably export GOPROXY in go-module_set_globals.
>
> William
>
Yes, go-module_set_globals is a nice place to consolidate all of the
exports. Great idea!
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-dev] Re: [PATCH] go-module.eclass: add functions for use in custom src_unpack phase
On 5/19/21 1:45 PM, Zac Medico wrote:
>> +# @FUNCTION: go-module_setup_proxy
>> +# @DESCRIPTION:
>> +# If your ebuild redefines src_unpack and uses EGO_SUM you need to call
>> +# this function in src_unpack.
>> +# It sets up the go module proxy in the appropriate location and exports
>> +# the GOPROXY environment variable so that go calls will be able to
>> +# locate the proxy directory.
>> +go-module_setup_proxy() {
>> +# shellcheck disable=SC2120
>> +debug-print-function "${FUNCNAME}" "$@"
>> +
>> +if [[ ! ${_GO_MODULE_SET_GLOBALS_CALLED} ]]; then
>> +die "go-module_set_globals must be called in global scope"
>> +fi
>> +
>> +local goproxy_dir="${T}/go-proxy"
>> +mkdir -p "${goproxy_dir}" || die
>> +
>> +# For each Golang module distfile, look up where it's supposed to go and
>> +# symlink it into place.
>> +local f
>> +local goproxy_mod_dir
>> +for f in ${A}; do
>> +goproxy_mod_path="${_GOMODULE_GOSUM_REVERSE_MAP["${f}"]}"
>> +if [[ -n "${goproxy_mod_path}" ]]; then
>> +debug-print-function "Populating go proxy for
>> ${goproxy_mod_path}"
>> +# Build symlink hierarchy
>> +goproxy_mod_dir=$( dirname
>> "${goproxy_dir}"/"${goproxy_mod_path}" )
>> +mkdir -p "${goproxy_mod_dir}" || die
>> +ln -sf "${DISTDIR}"/"${f}"
>> "${goproxy_dir}/${goproxy_mod_path}" ||
>> +die "Failed to ln"
>> +local v=${goproxy_mod_path}
>> +v="${v%.mod}"
>> +v="${v%.zip}"
>> +v="${v//*\/}"
>> +_go-module_gosum_synthesize_files "${goproxy_mod_dir}"
>> "${v}"
>> +fi
>> +done
>> +export GOPROXY="file://${goproxy_dir}"
>> +
>> +# Validate the gosum now
>> +_go-module_src_unpack_verify_gosum
>> +}
>> +
>> # @FUNCTION: go-module_src_unpack
>> # @DESCRIPTION:
>> # If EGO_SUM is set, unpack the base tarball(s) and set up the
>>
>
> The go-module_setup_proxy function solves bug 790851 nicely, since
> sys-cluster/k3s ebuilds can call that instead of go-module_src_unpack.
I do have one criticism of the go-module_setup_proxy, which is that it
relies on the side-effect of the GOPROXY export for its operation. We
can instead echo the GOPROXY value to stdout and force the caller to
export it themselves, and provide a convenience wrapper function which
works based on side-effects.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-dev] Re: [PATCH] go-module.eclass: add functions for use in custom src_unpack phase
On 5/19/21 12:48 PM, William Hubbs wrote:
> If an ebuild uses EGO_SUM and needs to define a custom src_unpack phase,
> these functions will make that easier.
>
> go-module_setup_proxy is used to create a local file proxy of the
> dependencies listed in EGO_SUM and go-module_filter_proxy is used to
> create a new ${A} with the EGO_SUM_SRC_URI values removed.
>
> Signed-off-by: William Hubbs
> ---
> eclass/go-module.eclass | 69 +
> 1 file changed, 69 insertions(+)
>
> diff --git a/eclass/go-module.eclass b/eclass/go-module.eclass
> index c9a7ab12eaf..80e1f711215 100644
> --- a/eclass/go-module.eclass
> +++ b/eclass/go-module.eclass
> @@ -236,6 +236,75 @@ go-module_set_globals() {
> _GO_MODULE_SET_GLOBALS_CALLED=1
> }
>
> +# @FUNCTION: go-module_filter_proxy
> +# @DESCRIPTION:
> +# If your ebuild redefines src_unpack and uses EGO_SUM, use the return
> +# value of this function in place of ${A} in your src_unpack function.
> +# It filters the EGO_SUM_SRC_URI values out of SRC_URI.
> +go-module_filter_proxy() {
> + # shellcheck disable=SC2120
> + debug-print-function "${FUNCNAME}" "$@"
> +
> + if [[ ! ${_GO_MODULE_SET_GLOBALS_CALLED} ]]; then
> + die "go-module_set_globals must be called in global scope"
> + fi
> +
> + # Skip golang modules and add the rest of the entries to a local
> + # array.
> + local f
> + local -a src
> + for f in ${A}; do
> + if [[ -z ${_GOMODULE_GOSUM_REVERSE_MAP["${f}"]} ]]; then
> + src+=("${f}")
> + fi
> + done
> + echo "${src[@]}"
> +}
While this go-module_filter_proxy might seem like a convenient way for
an ebuild to list distfiles which are not derived from EGO_SUM, I think
it's not the responsibility of go-module.eclass to provide this
information, therefore it's probably better to force the ebuild
developer to provide their own mechanism to track those files if needed,
rather than add a superfluous function to the eclass.
> +# @FUNCTION: go-module_setup_proxy
> +# @DESCRIPTION:
> +# If your ebuild redefines src_unpack and uses EGO_SUM you need to call
> +# this function in src_unpack.
> +# It sets up the go module proxy in the appropriate location and exports
> +# the GOPROXY environment variable so that go calls will be able to
> +# locate the proxy directory.
> +go-module_setup_proxy() {
> + # shellcheck disable=SC2120
> + debug-print-function "${FUNCNAME}" "$@"
> +
> + if [[ ! ${_GO_MODULE_SET_GLOBALS_CALLED} ]]; then
> + die "go-module_set_globals must be called in global scope"
> + fi
> +
> + local goproxy_dir="${T}/go-proxy"
> + mkdir -p "${goproxy_dir}" || die
> +
> + # For each Golang module distfile, look up where it's supposed to go and
> + # symlink it into place.
> + local f
> + local goproxy_mod_dir
> + for f in ${A}; do
> + goproxy_mod_path="${_GOMODULE_GOSUM_REVERSE_MAP["${f}"]}"
> + if [[ -n "${goproxy_mod_path}" ]]; then
> + debug-print-function "Populating go proxy for
> ${goproxy_mod_path}"
> + # Build symlink hierarchy
> + goproxy_mod_dir=$( dirname
> "${goproxy_dir}"/"${goproxy_mod_path}" )
> + mkdir -p "${goproxy_mod_dir}" || die
> + ln -sf "${DISTDIR}"/"${f}"
> "${goproxy_dir}/${goproxy_mod_path}" ||
> + die "Failed to ln"
> + local v=${goproxy_mod_path}
> + v="${v%.mod}"
> + v="${v%.zip}"
> + v="${v//*\/}"
> + _go-module_gosum_synthesize_files "${goproxy_mod_dir}"
> "${v}"
> + fi
> + done
> + export GOPROXY="file://${goproxy_dir}"
> +
> + # Validate the gosum now
> + _go-module_src_unpack_verify_gosum
> +}
> +
> # @FUNCTION: go-module_src_unpack
> # @DESCRIPTION:
> # If EGO_SUM is set, unpack the base tarball(s) and set up the
>
The go-module_setup_proxy function solves bug 790851 nicely, since
sys-cluster/k3s ebuilds can call that instead of go-module_src_unpack.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH v2 2/2] ebuild.5: Add eapply documentation
On 4/10/21 6:23 PM, Nekun wrote: > Signed-off-by: Nekun > --- > man/ebuild.5 | 20 > 1 file changed, 20 insertions(+) Thank you! I've merged these and noted it on the bug here: https://bugs.gentoo.org/698244#c1 Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] emerge: use parse_intermixed_args when available (bug 784566)
The included unit test case previously failed with this error:
emerge: error: unrecognized arguments: dev-libs/A
Bug: https://bugs.gentoo.org/784566
Signed-off-by: Zac Medico
---
lib/_emerge/main.py | 2 +-
lib/portage/tests/emerge/test_simple.py | 10 +-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 31e690584..191be6479 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -834,7 +834,7 @@ def parse_opts(tmpcmdline, silent=False):
tmpcmdline = insert_optional_args(tmpcmdline)
- myoptions = parser.parse_args(args=tmpcmdline)
+ myoptions = getattr(parser, "parse_intermixed_args",
parser.parse_args)(args=tmpcmdline)
if myoptions.alert in true_y:
myoptions.alert = True
diff --git a/lib/portage/tests/emerge/test_simple.py
b/lib/portage/tests/emerge/test_simple.py
index 6e282337f..5b110407f 100644
--- a/lib/portage/tests/emerge/test_simple.py
+++ b/lib/portage/tests/emerge/test_simple.py
@@ -1,6 +1,7 @@
# Copyright 2011-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import argparse
import subprocess
import portage
@@ -289,7 +290,14 @@ call_has_and_best_version() {
port=binhost_server.server_port,
path=binhost_remote_path)
- test_commands = (
+ test_commands = ()
+
+ if hasattr(argparse.ArgumentParser, "parse_intermixed_args"):
+ test_commands += (
+ emerge_cmd + ("--oneshot", "dev-libs/A", "-v",
"dev-libs/A"),
+ )
+
+ test_commands += (
emerge_cmd + ("--usepkgonly", "--root", cross_root,
"--quickpkg-direct=y", "--quickpkg-direct-root", "/", "dev-libs/A"),
emerge_cmd + ("--usepkgonly", "--quickpkg-direct=y",
"--quickpkg-direct-root", cross_root, "dev-libs/A"),
env_update_cmd,
--
2.26.2
Re: [gentoo-portage-dev] profile-formats not respected ?
On 4/20/21 5:03 AM, Joakim Tjernlund wrote:
> On Mon, 2021-04-19 at 14:10 -0700, Zac Medico wrote:
>> On 4/19/21 6:36 AM, Joakim Tjernlund wrote:
>>> I got an embedded ppc32 system which I build in a QEMU user chroot and I
>>> cannot
>>> make the profile-formats = portage-2 profile-bashrcs profile-set in my own
>>> profiles layout.conf
>>> work for me.
>>> Seems like portage just ignores this setting and I cannot understand why.
>>> Any pointers?
>>>
>>> Â Jocke
>>>
>>
>> Hopefully this command will provide a clue:
>>
>> python -c 'import portage;
>> print(portage.settings._locations_manager.profiles_complex)'
>
> Got some progress. I got a profile.bashrc:
> .../my-overlay/profiles/cusfpv3/profile.bashrc
> where I setup PKG_INSTALL_MASK/INSTALL_MASK and this does not work in
> portage-3.0.18. I can echo the vars. and see them set but portage ignores
> these settings.
> This system was previously using a very old portage, 2.3.76 and that was fine.
>
>
> Another strange thing is my own set, i need to keep that at
> .../my-overlay/{sets, sets.conf}
> In our amd64 DE profile I can have my sets at
> .../my-overlay/profiles/infinera/sets , why is that?
A setting like this in /etc/portage/sets.conf would do it:
[my-overlay-profile-sets]
class = portage.sets.files.StaticFileSet
multiset = true
directory = /my-overlay/profiles/infinera/sets
world-candidate = true
> Anyhow, here is my profile printout:
> ./print-port.py
> (_profile_node(location='/usr/portage/profiles/base',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/default/linux',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',),
> eapi='5', allow_build_id=False, show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/arch/base',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/arch/powerpc',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/default/linux/powerpc',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5',
> allow_build_id=False, show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/arch/base',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/arch/powerpc',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/arch/powerpc/ppc32',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5',
> allow_build_id=False, show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/default/linux/powerpc/ppc32',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/releases',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/releases/17.0',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/default/linux/powerpc/ppc32/17.0',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=True),
> _profile_node(location='/usr/local/portage/tmv3-target-
> overlay/profiles/cusfpv3', portage1_directories=True, user_config=False,
> profile_formats=('profile-bashrcs', 'portage-2', 'profile-set'), eapi='5',
> allow_build_id=False,
> show_deprecated_warning=True), _profile_node(location='/etc/portage/profile',
> portage1_directories=True, user_config=True,
> profile_formats=('profile-bashrcs', 'profile-set'),
> eapi=None, allow_build_id=True, show_deprecated_warning=False))
>
Are these effective profile_formats now consistent with your
expectations, and do they now behave as you would expect?
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] profile-formats not respected ?
On 4/19/21 6:36 AM, Joakim Tjernlund wrote: > I got an embedded ppc32 system which I build in a QEMU user chroot and I > cannot > make the profile-formats = portage-2 profile-bashrcs profile-set in my own > profiles layout.conf > work for me. > Seems like portage just ignores this setting and I cannot understand why. > Any pointers? > > Jocke > Hopefully this command will provide a clue: python -c 'import portage; print(portage.settings._locations_manager.profiles_complex)' -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] make.conf.5: Sugest PORTAGE_LOG_FILTER_FILE_CMD supervisor for cat fallback (bug 781854)
If PORTAGE_LOG_FILTER_FILE_CMD fails after exec, then output will be lost. Therefore, suggest to use bash as a supervisor, with fallback to cat. Bug: https://bugs.gentoo.org/781854 Signed-off-by: Zac Medico --- man/make.conf.5 | 15 +-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/man/make.conf.5 b/man/make.conf.5 index 8d551c95e..8a4a2ae30 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -1,4 +1,4 @@ -.TH "MAKE.CONF" "5" "Feb 2021" "Portage VERSION" "Portage" +.TH "MAKE.CONF" "5" "Apr 2021" "Portage VERSION" "Portage" .SH "NAME" make.conf \- custom settings for Portage .SH "SYNOPSIS" @@ -998,7 +998,18 @@ will set idle io priority. For more information about ionice, see .B PORTAGE_LOG_FILTER_FILE_CMD This variable specifies a command that filters build log output to a log file. In order to filter ANSI escape codes from build logs, -\fBansifilter\fR(1) is a convenient setting for this variable. +\fBansifilter\fR(1) is a convenient setting for this variable. Generally, +PORTAGE_LOG_FILTER_FILE_CMD should include a supervisor that falls back +to cat if the real filter command fails after exec. For example, a +supervisor is needed for ansifilter, in case it fails after exec due to +a problem resolving libstdc++ during a gcc upgrade. +.br +.I Example: +.nf +# Use bash as a supervisor, for fallback to cat if asifilter fails +# after exec due to a problem resolving libstdc++ during a gcc upgrade. +PORTAGE_LOG_FILTER_FILE_CMD="bash -c \\"ansifilter; exec cat\\"" +.fi .TP .B PORTAGE_LOGDIR This variable defines the directory in which per\-ebuild logs are kept. -- 2.26.2
Re: [gentoo-portage-dev] Implement new userpatch feature in existing eclass?
On 4/4/21 10:54 AM, Nekun wrote:
> Hi all.
>
> Recently, I start working on optional atom specifiers feature in
> userpatch facility: if package directory name starts with percent sign,
> following word threated as a regular Portage atom, e.g
> "/etc/portage/patches/sys-kernel/%<=gentoo-sources-5.4" ==
> "<=sys-kernel/gentoo-sources-5.4". This might be very useful in cases
> when patches applied to minor updates, but major update breaks it (e.g.,
> in Linux kernel), so I want to specify smth like "=gentoo-sources-5.4*".
> I added new command in portageq to match two atoms and call it from
> eapply_user function in phase-function.sh, in same manner as
> has_version/best_version are called it. But recently I found that
> eapply_user implemented in Portage only in EAPI 6, and there is its
> predecessor, epatch_user, implemented in epatch.eclass. So, ebuilds with
> EAPI<6 (I found 4463 in last gentoo snapshot) will ignore new "atomic"
> patch directories. Obviously, this is rather confusing, unacceptable
> behaviour.
>
> Can I patch epatch.eclass in gentoo repository to implement new
> userpatch facility for older EAPIs? I guess that EAPI version is
> considered as stable, unchangeable behaviour of all functions, but in
> other side, this feature doesn't changes anything existing: old
> userpatch semantics preserves and order of applying
> (${P}-${PR},${P},${PN}) not changed, seeking for atoms added at tail.
>
Today, I count only 2445, or 8.3% of ebuilds have EAPI 5.
I imagine that the migration is moving along, since we deprecated EAPI 5
on 2020-11-26 here:
https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=b2e281bb698eb93704e1987dc4df1cf2dd3c2cff
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] Implement new userpatch feature in existing eclass?
On 4/4/21 10:54 AM, Nekun wrote:
> Hi all.
>
> Recently, I start working on optional atom specifiers feature in
> userpatch facility: if package directory name starts with percent sign,
> following word threated as a regular Portage atom, e.g
> "/etc/portage/patches/sys-kernel/%<=gentoo-sources-5.4" ==
> "<=sys-kernel/gentoo-sources-5.4". This might be very useful in cases
> when patches applied to minor updates, but major update breaks it (e.g.,
> in Linux kernel), so I want to specify smth like "=gentoo-sources-5.4*".
> I added new command in portageq to match two atoms and call it from
> eapply_user function in phase-function.sh, in same manner as
> has_version/best_version are called it. But recently I found that
> eapply_user implemented in Portage only in EAPI 6, and there is its
> predecessor, epatch_user, implemented in epatch.eclass. So, ebuilds with
> EAPI<6 (I found 4463 in last gentoo snapshot) will ignore new "atomic"
> patch directories. Obviously, this is rather confusing, unacceptable
> behaviour.
>
> Can I patch epatch.eclass in gentoo repository to implement new
> userpatch facility for older EAPIs? I guess that EAPI version is
> considered as stable, unchangeable behaviour of all functions, but in
> other side, this feature doesn't changes anything existing: old
> userpatch semantics preserves and order of applying
> (${P}-${PR},${P},${PN}) not changed, seeking for atoms added at tail.
>
Today, I count only 2445, or 8.3% of ebuilds have EAPI 5.
I imagine that the migration is moving along, since we deprecated EAPI 5
on 2020-11-26 here:
https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=b2e281bb698eb93704e1987dc4df1cf2dd3c2cff
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Use atomic_ofstream as Context Manager i.e., with-statement contexts
On 3/8/21 11:25 PM, Florian Schmaus wrote: > With [1: e93e6d65fa1c] atomic_ofstream became a Context Manager. This > commit transforms three further call sites of atomic_ofstream() to use > with-statement contexts for easier readability and increased > robustness against resource leaks. > > 1: e93e6d65fa1ca75f676a227f7918f8b6d747425c >Make atomic_ofstream a Context Manager > > Signed-off-by: Florian Schmaus > --- > lib/_emerge/BlockerCache.py| 6 +++--- > lib/portage/dbapi/_VdbMetadataDelta.py | 11 +-- > lib/portage/dbapi/vartree.py | 6 +++--- Merged. Thank you! https://gitweb.gentoo.org/proj/portage.git/commit/?id=8879cef9006d2277453aaee407c234a2d1bc47ba -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Mark EAPIs "4-python" and "5-progress" as deprecated
On 3/4/21 11:35 AM, Matt Turner wrote: > Signed-off-by: Matt Turner > --- > I've asked Arfrever multiple times if these are still used anywhere, and > he seemingly has not responded intentionally. > > According to https://bugs.gentoo.org/174536#c27 these EAPIs were only > used in Arfrever's personal overlay, and even in 2012 there were > questions about why they were supported in portage. > > The "Progress Overlay" does contain ebuilds using these EAPIs but it has > not been updated since 2018 and doesn't look like it is useful at this > point. > > lib/portage/__init__.py | 8 > 1 file changed, 4 insertions(+), 4 deletions(-) > > diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py > index 24c9d8b89..184db6ae2 100644 > --- a/lib/portage/__init__.py > +++ b/lib/portage/__init__.py > @@ -465,16 +465,16 @@ def abssymlink(symlink, target=None): > _doebuild_manifest_exempt_depend = 0 > > _testing_eapis = frozenset([ > - "4-python", > - "5-progress", > ]) > _deprecated_eapis = frozenset([ > + "3_pre1", > + "3_pre2", > "4_pre1", > + "4-python", > "4-slot-abi", > - "3_pre2", > - "3_pre1", > "5_pre1", > "5_pre2", > + "5-progress", > "6_pre1", > "7_pre1", > ]) > Merged, thanks! https://gitweb.gentoo.org/proj/portage.git/commit/?id=67cf9c2b05042de37f36f5b6840c450128a065bd -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Use asyncio.subprocess.Process directly
On 3/4/21 11:24 AM, Matt Turner wrote:
> With no need to support Python 2, we can remove our private
> implementation.
>
> Signed-off-by: Matt Turner
> ---
> I don't know how to test this. I intentionally broke the return value of
> create_subprocess_exec and didn't see any bad results.
>
> lib/portage/util/futures/_asyncio/__init__.py | 8 +-
> lib/portage/util/futures/_asyncio/process.py | 116 --
> 2 files changed, 4 insertions(+), 120 deletions(-)
> delete mode 100644 lib/portage/util/futures/_asyncio/process.py
Merged, thanks!
https://gitweb.gentoo.org/proj/portage.git/commit/?id=1e843f853a9afe82d599e6ab09064147ddc1d271
> diff --git a/lib/portage/util/futures/_asyncio/__init__.py
> b/lib/portage/util/futures/_asyncio/__init__.py
> index 5590963f1..207e7205d 100644
> --- a/lib/portage/util/futures/_asyncio/__init__.py
> +++ b/lib/portage/util/futures/_asyncio/__init__.py
> @@ -25,6 +25,7 @@ import types
> import weakref
>
> import asyncio as _real_asyncio
> +from asyncio.subprocess import Process
>
> try:
> import threading
> @@ -138,7 +138,7 @@ def create_subprocess_exec(*args, **kwargs):
>
> result = loop.create_future()
>
> - result.set_result(_Process(subprocess.Popen(
> + result.set_result(Process(subprocess.Popen(
> args,
> stdin=kwargs.pop('stdin', None),
> stdout=kwargs.pop('stdout', None),
The above area is actually no longer used, since we should always have a
_AsyncioEventLoop instance here, and we can remove the EventLoop class now.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH 3/3] lib: Remove outdated Python 2 comments
On 3/4/21 11:23 AM, Matt Turner wrote:
> Fixes: 788c0e8bb ("Remove from __future__ import unicode_literals")
> Signed-off-by: Matt Turner
> ---
> bin/egencache | 2 --
> lib/_emerge/Package.py | 9 -
> lib/_emerge/Scheduler.py| 2 --
> lib/_emerge/UseFlagDisplay.py | 2 --
> lib/_emerge/resolver/output.py | 2 --
> lib/portage/cache/flat_hash.py | 3 ---
> lib/portage/tests/unicode/test_string_format.py | 9 -
> lib/portage/util/digraph.py | 3 ---
> 8 files changed, 32 deletions(-)
Series looks good. Merged. Thanks!
https://gitweb.gentoo.org/proj/portage.git/commit/?id=9003c5201c6503ddad9237bcffbc6f775567661b
https://gitweb.gentoo.org/proj/portage.git/commit/?id=af100c65ebf7fd84307a84819602a934ebb0741c
https://gitweb.gentoo.org/proj/portage.git/commit/?id=21c6c0c1088ded78397594bfd78102361f8d837b
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-dev] Package up for grabs: dev-python/python-backoff
We have and ebuild for this package up for grabs: https://pypi.org/project/backoff/ No open bugs and no reverse dependencies: https://bugs.gentoo.org/buglist.cgi?quicksearch=python-backoff -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] emerge: make --binpkg-respect-use=y imply --autounmask-use=n
If --binpkg-respect-use=y is given explicitly, then it implies
--autounmask-use=n, because these options naturally oppose
eachother.
Bug: https://bugs.gentoo.org/773469
Signed-off-by: Zac Medico
---
lib/_emerge/create_depgraph_params.py | 22 ++---
lib/portage/tests/resolver/test_useflags.py | 20 +--
man/emerge.1| 11 +++
3 files changed, 32 insertions(+), 21 deletions(-)
diff --git a/lib/_emerge/create_depgraph_params.py
b/lib/_emerge/create_depgraph_params.py
index 25dd2a1b4..267600fb6 100644
--- a/lib/_emerge/create_depgraph_params.py
+++ b/lib/_emerge/create_depgraph_params.py
@@ -41,12 +41,22 @@ def create_depgraph_params(myopts, myaction):
# binpkg_changed_deps: reject binary packages with outdated deps
myparams = {"recurse" : True}
+ binpkg_respect_use = myopts.get("--binpkg-respect-use")
+ if binpkg_respect_use is not None:
+ myparams["binpkg_respect_use"] = binpkg_respect_use
+ elif "--usepkgonly" not in myopts:
+ # If --binpkg-respect-use is not explicitly specified, we enable
+ # the behavior automatically (like requested in bug #297549), as
+ # long as it doesn't strongly conflict with other options that
+ # have been specified.
+ myparams["binpkg_respect_use"] = "auto"
+
autounmask_keep_keywords = myopts.get("--autounmask-keep-keywords")
autounmask_keep_masks = myopts.get("--autounmask-keep-masks")
autounmask = myopts.get("--autounmask")
autounmask_license = myopts.get('--autounmask-license', 'y' if
autounmask is True else 'n')
- autounmask_use = myopts.get('--autounmask-use')
+ autounmask_use = 'n' if myparams.get('binpkg_respect_use') == 'y' else
myopts.get('--autounmask-use')
if autounmask == 'n':
autounmask = False
else:
@@ -153,16 +163,6 @@ def create_depgraph_params(myopts, myaction):
'--update' in myopts:
myparams['rebuilt_binaries'] = True
- binpkg_respect_use = myopts.get('--binpkg-respect-use')
- if binpkg_respect_use is not None:
- myparams['binpkg_respect_use'] = binpkg_respect_use
- elif '--usepkgonly' not in myopts:
- # If --binpkg-respect-use is not explicitly specified, we enable
- # the behavior automatically (like requested in bug #297549), as
- # long as it doesn't strongly conflict with other options that
- # have been specified.
- myparams['binpkg_respect_use'] = 'auto'
-
binpkg_changed_deps = myopts.get('--binpkg-changed-deps')
if binpkg_changed_deps is not None:
myparams['binpkg_changed_deps'] = binpkg_changed_deps
diff --git a/lib/portage/tests/resolver/test_useflags.py
b/lib/portage/tests/resolver/test_useflags.py
index d66da0866..b799e62ff 100644
--- a/lib/portage/tests/resolver/test_useflags.py
+++ b/lib/portage/tests/resolver/test_useflags.py
@@ -46,15 +46,23 @@ class UseFlagsTestCase(TestCase):
success = True,
mergelist = ["dev-libs/A-1"]),
- # In the unit test case for bug 773469, the
--autounmask-backtrack option
- # is needed in order to trigger the
--binpkg-respect-use=y behavior that
- # appears confusingly similar to --binpkg-respect-use=n
behavior.
+ # For bug 773469, we wanted --binpkg-respect-use=y to
trigger a
+ # slot collision. Instead, a combination of default
--autounmask-use
+ # combined with --autounmask-backtrack=y from
EMERGE_DEFAULT_OPTS
+ # triggered this behavior which appeared confusingly
similar to
+ #--binpkg-respect-use=n behavior.
+ #ResolverPlaygroundTestCase(
+ # ["dev-libs/C", "dev-libs/D"],
+ # options={"--usepkg": True,
"--binpkg-respect-use": "y", "--autounmask-backtrack": "y"},
+ # success=True,
+ # use_changes={"dev-libs/C-1": {"abi_x86_32":
True}},
+ # mergelist=["[binary]dev-libs/C-1",
"[binary]dev-libs/D-1"],
ResolverPlaygroundTestCase(
["dev-libs/C", "dev-libs/D"],
options={"--usepkg": True,
"--binpkg-respect-use": "y", "--autounmask-backtrack": "y"},
- success=True,
-
[gentoo-portage-dev] Planning to publish portage releases on pypi
I'd like to begin publishing portage releases on pypi here: https://pypi.org/project/portage It won't allow me to create the the project, so I've opened this ticket to claim it: https://github.com/pypa/pypi-support/issues/934 -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v3] emirrordist: add --content-db option required for content-hash layout (bug 756778)
Add a --content-db option which is required for the content-hash
layout because its file listings return content digests instead of
distfile names.
The content db serves to translate content digests to distfiles
names, and distfiles names to content digests. All keys have a
prefix separated by a colon. For digest keys, the prefix is the
hash algorithm name. For filename keys, the prefix is "filename".
The value associated with a digest key is a set of file names. The
value associated with a distfile key is a set of content revisions.
Each content revision is expressed as a dictionary of digests which
is suitable for construction of a DistfileName instance.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico
---
[PATCH v3] changed the value associated with a digest key is a set
of file name, and fixed ContentDB.remove to preserved independent
references to identical content (like removing one of multiple
hardlinks).
lib/portage/_emirrordist/Config.py | 8 +-
lib/portage/_emirrordist/ContentDB.py| 178 +++
lib/portage/_emirrordist/DeletionIterator.py | 25 ++-
lib/portage/_emirrordist/DeletionTask.py | 8 +
lib/portage/_emirrordist/FetchTask.py| 5 +-
lib/portage/_emirrordist/main.py | 15 +-
lib/portage/package/ebuild/fetch.py | 8 +-
lib/portage/tests/ebuild/test_fetch.py | 14 ++
man/emirrordist.1| 6 +-
9 files changed, 256 insertions(+), 11 deletions(-)
create mode 100644 lib/portage/_emirrordist/ContentDB.py
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index 4bee4f45e..cfe944040 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -10,6 +10,7 @@ import time
from portage import os
from portage.package.ebuild.fetch import MirrorLayoutConfig
from portage.util import grabdict, grablines
+from .ContentDB import ContentDB
class Config:
def __init__(self, options, portdb, event_loop):
@@ -65,6 +66,11 @@ class Config:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
+ self.content_db = None
+ if options.content_db is not None:
+ self.content_db = ContentDB(self._open_shelve(
+ options.content_db, 'content'))
+
self.deletion_db = None
if options.deletion_db is not None:
self.deletion_db = self._open_shelve(
diff --git a/lib/portage/_emirrordist/ContentDB.py
b/lib/portage/_emirrordist/ContentDB.py
new file mode 100644
index 0..7084cecff
--- /dev/null
+++ b/lib/portage/_emirrordist/ContentDB.py
@@ -0,0 +1,178 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import operator
+import shelve
+import typing
+
+from portage.package.ebuild.fetch import DistfileName
+
+
+class ContentDB:
+ """
+ The content db serves to translate content digests to distfiles
+ names, and distfiles names to content digests. All keys have a
+ prefix separated by a colon. For digest keys, the prefix is the
+ hash algorithm name. For filename keys, the prefix is "filename".
+
+ The value associated with a digest key is a set of file names. The
+ value associated with a distfile key is a set of content revisions.
+ Each content revision is expressed as a dictionary of digests which
+ is suitable for construction of a DistfileName instance.
+ """
+
+ def __init__(self, shelve_instance: shelve.Shelf):
+ self._shelve = shelve_instance
+
+ def add(self, filename: DistfileName):
+ """
+ Add file name and digests.
+
+ @param filename: file name with digests attribute
+ """
+ distfile_str = str(filename)
+ distfile_key = "filename:{}".format(distfile_str)
+ for k, v in filename.digests.items():
+ if k != "size":
+ digest_key = "{}:{}".format(k, v).lower()
+ try:
+ digest_files = self._shelve[digest_key]
+ except KeyError:
+ digest_files = set()
+ digest_files.add(distfile_str)
+ self._shelve[digest_key] = digest_files
+ try:
+ content_revisions = self._shelve[distfile_key]
+
[gentoo-portage-dev] [PATCH] make.globals: make FEATURES=-binpkg-multi-instance sticky for existing installs
Add a _compat_upgrade.binpkg_multi_instance script that the ebuild
can call in pkg_preinst in order to maintain a backward-compatible
FEATURES=-binpkg-multi-instance default on existing installs where
enabling binpkg-multi-instance could cause disruption.
Bug: https://bugs.gentoo.org/772785
Signed-off-by: Zac Medico
---
.../_compat_upgrade/binpkg_multi_instance.py | 33 +++
1 file changed, 33 insertions(+)
create mode 100644 lib/portage/_compat_upgrade/binpkg_multi_instance.py
diff --git a/lib/portage/_compat_upgrade/binpkg_multi_instance.py
b/lib/portage/_compat_upgrade/binpkg_multi_instance.py
new file mode 100644
index 0..bfeb72baa
--- /dev/null
+++ b/lib/portage/_compat_upgrade/binpkg_multi_instance.py
@@ -0,0 +1,33 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import portage
+from portage import os
+from portage.const import GLOBAL_CONFIG_PATH
+
+COMPAT_FEATURES = 'FEATURES="${FEATURES} -binpkg-multi-instance"'
+
+
+def main():
+ """
+ If the current installation is still has binpkg-multi-instance
+ disabled, then patch make.globals inside ${ED} to maintain backward
+ compatibility. This is intended to be called from the ebuild as
+ follows:
+
+ pkg_preinst() {
+ python_setup
+ env -u FEATURES
+
PYTHONPATH="${D%/}$(python_get_sitedir)${PYTHONPATH:+:${PYTHONPATH}}" \
+ "${PYTHON}" -m
portage._compat_upgrade.binpkg_multi_instance || die
+ }
+ """
+ if 'binpkg-multi-instance' not in portage.settings.features:
+ portage.output.EOutput().einfo('Setting make.globals default {}
for backward compatibility'.format(COMPAT_FEATURES))
+ config_path = os.path.join(os.environ['ED'],
GLOBAL_CONFIG_PATH.lstrip(os.sep), 'make.globals')
+ with open(config_path, 'at') as f:
+ f.write("{}\n".format(COMPAT_FEATURES))
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
[gentoo-portage-dev] [PATCH v2] emirrordist: add --content-db option required for content-hash layout (bug 756778)
Add a --content-db option which is required for the content-hash
layout because its file listings return content digests instead of
distfile names.
The content db serves to translate content digests to distfiles
names, and distfiles names to content digests. All keys have a
prefix separated by a colon. For digest keys, the prefix is the
hash algorithm name. For filename keys, the prefix is "filename".
The value associated with a digest key is a plain filename. The
value associated with a distfile key is a set of content revisions.
Each content revision is expressed as a dictionary of digests which
is suitable for construction of a DistfileName instance.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico
---
[PATCH v2] Split out ContentDB class and associate distfile key
with a set of content revisions, where each content revision is
expressed as a dictionary of digests.
lib/portage/_emirrordist/Config.py | 8 +-
lib/portage/_emirrordist/ContentDB.py| 158 +++
lib/portage/_emirrordist/DeletionIterator.py | 25 ++-
lib/portage/_emirrordist/DeletionTask.py | 8 +
lib/portage/_emirrordist/FetchTask.py| 5 +-
lib/portage/_emirrordist/main.py | 15 +-
lib/portage/tests/ebuild/test_fetch.py | 14 ++
man/emirrordist.1| 6 +-
8 files changed, 232 insertions(+), 7 deletions(-)
create mode 100644 lib/portage/_emirrordist/ContentDB.py
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index 4bee4f45e..cfe944040 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -10,6 +10,7 @@ import time
from portage import os
from portage.package.ebuild.fetch import MirrorLayoutConfig
from portage.util import grabdict, grablines
+from .ContentDB import ContentDB
class Config:
def __init__(self, options, portdb, event_loop):
@@ -65,6 +66,11 @@ class Config:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
+ self.content_db = None
+ if options.content_db is not None:
+ self.content_db = ContentDB(self._open_shelve(
+ options.content_db, 'content'))
+
self.deletion_db = None
if options.deletion_db is not None:
self.deletion_db = self._open_shelve(
diff --git a/lib/portage/_emirrordist/ContentDB.py
b/lib/portage/_emirrordist/ContentDB.py
new file mode 100644
index 0..60e6ef39d
--- /dev/null
+++ b/lib/portage/_emirrordist/ContentDB.py
@@ -0,0 +1,158 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import operator
+import shelve
+import typing
+
+from portage.package.ebuild.fetch import DistfileName
+
+
+class ContentDB:
+ """
+ The content db serves to translate content digests to distfiles
+ names, and distfiles names to content digests. All keys have a
+ prefix separated by a colon. For digest keys, the prefix is the
+ hash algorithm name. For filename keys, the prefix is "filename".
+
+ The value associated with a digest key is a plain filename. The
+ value associated with a distfile key is a set of content revisions.
+ Each content revision is expressed as a dictionary of digests which
+ is suitable for construction of a DistfileName instance.
+ """
+
+ def __init__(self, shelve_instance: shelve.Shelf):
+ self._shelve = shelve_instance
+
+ def add(self, filename: DistfileName):
+ """
+ Add file name and digests.
+
+ @param filename: file name with digests attribute
+ """
+ distfile_str = str(filename)
+ distfile_key = "filename:{}".format(distfile_str)
+ for k, v in filename.digests.items():
+ if k != "size":
+ self._shelve["{}:{}".format(k, v).lower()] =
distfile_str
+ try:
+ content_revisions = self._shelve[distfile_key]
+ except KeyError:
+ content_revisions = set()
+
+ revision_key = tuple(
+ sorted(
+ (
+ (algo.lower(),
filename.digests[algo].lower())
+ for algo in filename.digests
+
[gentoo-portage-dev] [PATCH] emirrordist: add --content-db option required for content-hash layout (bug 756778)
Add a --content-db option which is required for the content-hash
layout because its file listings return content digests instead of
distfile names.
The content db includes a reverse mapping, for use during garbage
collection. All keys have a prefix separated by a colon. For digest
keys, the prefix is the hash algorithm name. For filename keys,
the prefix is "filename". The values for digest keys are plain
filenames, and the values for distfile keys are dictionaries
of digests suitable for construction of DistfileName instances.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico
---
lib/portage/_emirrordist/Config.py | 7 +++-
lib/portage/_emirrordist/DeletionIterator.py | 38 ++--
lib/portage/_emirrordist/DeletionTask.py | 26 ++
lib/portage/_emirrordist/FetchTask.py| 16 -
lib/portage/_emirrordist/main.py | 15 +++-
lib/portage/tests/ebuild/test_fetch.py | 14
man/emirrordist.1| 6 +++-
7 files changed, 116 insertions(+), 6 deletions(-)
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index 4bee4f45e..53f6582fe 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -65,6 +65,11 @@ class Config:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
+ self.content_db = None
+ if options.content_db is not None:
+ self.content_db = self._open_shelve(
+ options.content_db, 'content')
+
self.deletion_db = None
if options.deletion_db is not None:
self.deletion_db = self._open_shelve(
diff --git a/lib/portage/_emirrordist/DeletionIterator.py
b/lib/portage/_emirrordist/DeletionIterator.py
index 08985ed6c..24fb096bf 100644
--- a/lib/portage/_emirrordist/DeletionIterator.py
+++ b/lib/portage/_emirrordist/DeletionIterator.py
@@ -1,10 +1,12 @@
-# Copyright 2013-2019 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import logging
import stat
+import typing
from portage import os
+from portage.package.ebuild.fetch import DistfileName
from .DeletionTask import DeletionTask
class DeletionIterator:
@@ -12,6 +14,37 @@ class DeletionIterator:
def __init__(self, config):
self._config = config
+ def _map_filename(self, filename: typing.Union[str, DistfileName]) ->
typing.Union[str, DistfileName]:
+ """
+ Map a filename listed by the layout get_filenames method,
+ translating it from a content digest to a distfile name.
+ If filename is already a distfile name, then it will pass
+ through unchanged.
+
+ @param filename: A filename listed by layout get_filenames
+ @return: The distfile name, mapped from the corresponding
+ content digest when necessary
+ """
+ if not isinstance(filename, DistfileName):
+ if self._config.content_db is not None:
+ distfile_key = "filename:{}".format(filename)
+ try:
+ digests =
self._config.content_db[distfile_key]
+ except KeyError:
+ pass
+ else:
+ return DistfileName(filename,
digests=digests)
+ return DistfileName(filename)
+ if filename.digests and self._config.content_db is not None:
+ for k, v in filename.digests.items():
+ digest_key = "{}:{}".format(k, v).lower()
+ try:
+ distfile_str =
self._config.content_db[digest_key]
+ except KeyError:
+ continue
+ return DistfileName(distfile_str, digests={k:v})
+ return filename
+
def __iter__(self):
distdir = self._config.options.distfiles
file_owners = self._config.file_owners
@@ -22,7 +55,8 @@ class DeletionIterator:
start_time = self._config.start_time
distfiles_set = set()
for layout in self._config.layouts:
- distfiles_set.update(layout.get_filenames(distdir))
+
Re: [gentoo-dev] New project: binhost
On 2/24/21 2:29 AM, Zac Medico wrote: > For example, for 3 USE flags, up to 8 combinations will be indexed: > > IUSE="a b c installsources splitdebug" > SRC_URI=" > !a? !b? !c? ( mirror://binhost/24fe6bd377 ) > !a? !b? c? ( mirror://binhost/fbe14cbb02 ) > !a? b? !c? ( mirror://binhost/1dfff1f2ac ) > !a? b? c? ( mirror://binhost/ae60f2940d ) >a? !b? !c? ( mirror://binhost/2976e1acc0 ) >a? !b? c? ( mirror://binhost/f4809db70c ) >a? b? !c? ( mirror://binhost/ecd08466cf ) >a? b? c? ( mirror://binhost/0c00f33b2e ) > installsources? ( > !a? !b? !c? ( mirror://binhost/063a14d6c7 ) > !a? !b? c? ( mirror://binhost/f67c311625 ) > !a? b? !c? ( mirror://binhost/1dfff1f2ac ) > !a? b? c? ( mirror://binhost/17a673e16a ) > a? !b? !c? ( mirror://binhost/914d1cecfe ) > a? !b? c? ( mirror://binhost/ca18d86a2b ) > a? b? !c? ( mirror://binhost/6bce13471a ) > a? b? c? ( mirror://binhost/3a6bdcd228 ) > ) > splitdebug? ( > !a? !b? !c? ( mirror://binhost/29b2f38c41 ) > !a? !b? c? ( mirror://binhost/8adc9bef51 ) > !a? b? !c? ( mirror://binhost/954d2ce484 ) > !a? b? c? ( mirror://binhost/32a614aaca ) > a? !b? !c? ( mirror://binhost/3548a2302d ) > a? !b? c? ( mirror://binhost/e0c02cdc88 ) > a? b? !c? ( mirror://binhost/f9cbd3c181 ) > a? b? c? ( mirror://binhost/31d4c03474 ) > ) > " > > For installsources, we can automate deduplication, so that we can > distribute the same file content for multiple USE combinations when > appropriate. If all of the combinations have identical content, then it > will look like this: > > installsources? ( > !a? !b? !c? ( mirror://binhost/063a14d6c7 ) > !a? !b? c? ( mirror://binhost/063a14d6c7 ) > !a? b? !c? ( mirror://binhost/063a14d6c7 ) > !a? b? c? ( mirror://binhost/063a14d6c7 ) > a? !b? !c? ( mirror://binhost/063a14d6c7 ) > a? !b? c? ( mirror://binhost/063a14d6c7 ) > a? b? !c? ( mirror://binhost/063a14d6c7 ) > a? b? c? ( mirror://binhost/063a14d6c7 ) > ) > > In order to ensure that an ebin is not selected for a USE combination > that has not been built yet, combinations for existing builds will be > listed in REQUIRED_USE, like this: > > REQUIRED_USE=" > || ( > ( !a !b !c ) > ( !a !b c ) > ( !a b !c ) > ( !a b c ) > ( a !b !c ) > ( a !b c ) > ( a b !c ) > ( a b c ) > ) > " In https://bugs.gentoo.org/772380 I'm planning to implement a script that will take an existing $PKGDIR as input, and generate an "ebin" binhost as output. It will automatically split out pre-built content bundles for installsources and splitdebug as shown above. If there is more than one build for a particular package version and USE combination, then the script will choose the package instance with latest BUILD_TIME metadata (in alignment with FEATURES=binpkg-multi-instance). -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-dev] New project: binhost
On 2/23/21 12:33 PM, Zac Medico wrote: > On 2/23/21 12:05 PM, Zac Medico wrote: >> On 2/23/21 11:46 AM, Zac Medico wrote: >>> On 2/20/21 8:17 PM, Zac Medico wrote: >>>> IUSE_RUNTIME will obviously introduce conditionals in binary package >>>> dependencies, but we should welcome these conditionals because they will >>>> provide useful flexibility. >>>> >>>> I think IUSE_RUNTIME will be a very nice feature to have for project >>>> binhost, since it will allow binary package dependencies to behave with >>>> flexibility that more closely resembles the flexibility of ebuild >>>> dependencies. >>> >>> We can borrow paludis's notion of pbins [1] to generate ebuilds which >>> install pre-built content, and the generated ebuilds could have USE >>> flags that behave similarly to IUSE_RUNTIME in the sense that changes to >>> USE flags will result in different builds of pre-built content being >>> installed. A content-hash distfiles layout [2] could serve as a >>> convenient way to store separate builds of pre-built content for >>> multiple combinations of USE flags, and a generated ebuild would index >>> the build by USE flag combination. >>> >>> Also, for the generated ebuilds, we can generate USE flags to include >>> separate SRC_URI downloads for pre-built content to support things like >>> FEATURES=split-debug and FEATURES=install-sources. >> >> Note that all of this can use existing EAPI features, since everything new >> would be implemented in an ebuild generator that generates a single >> ebuild to index pre-built content from multiple binary package builds. >> >>> [1] https://paludis.exherbo.org/overview/pbins.html >>> [2] https://bugs.gentoo.org/756778 >> > > For generated ebuilds, we'll have a choice to model things as USE flags > or sub-packages. For example, content from FEATURES=split-debug and > FEATURES=install-sources content might make more sense to model as > sub-packages than USE flags. It makes more sense to generate a > sub-package when there is a need for the sub-package to have USE flags. > For example, a split-debug sub-package can have USE flags which index > pre-built content from builds for multiple USE flag combinations. > Similar USE flags could be useful for an install-sources sub-package if > the source code has patches which are conditional on USE flags. Since the generated ebuilds are inspired by pbins, we might call them "ebins". Once we have designed an ebin generation process that we're happy with, we should consider making it part of an EAPI, so that package managers can generate "ebins" on request. The EAPI should include ways to split out files and distribute them separately based on USE flags and/or sub-packages, so that binhost users can easily filter which files are installed based on USE configuration. We can automatically map user's splitdebug and installsources FEATURES settings into USE settings, in the same way that FEATURES=test automatically maps to USE=test. Each generated ebuild (ebin) will use its SRC_URI metadata to index builds for each USE flag combination for which a build exists. For example, for 3 USE flags, up to 8 combinations will be indexed: IUSE="a b c installsources splitdebug" SRC_URI=" !a? !b? !c? ( mirror://binhost/24fe6bd377 ) !a? !b? c? ( mirror://binhost/fbe14cbb02 ) !a? b? !c? ( mirror://binhost/1dfff1f2ac ) !a? b? c? ( mirror://binhost/ae60f2940d ) a? !b? !c? ( mirror://binhost/2976e1acc0 ) a? !b? c? ( mirror://binhost/f4809db70c ) a? b? !c? ( mirror://binhost/ecd08466cf ) a? b? c? ( mirror://binhost/0c00f33b2e ) installsources? ( !a? !b? !c? ( mirror://binhost/063a14d6c7 ) !a? !b? c? ( mirror://binhost/f67c311625 ) !a? b? !c? ( mirror://binhost/1dfff1f2ac ) !a? b? c? ( mirror://binhost/17a673e16a ) a? !b? !c? ( mirror://binhost/914d1cecfe ) a? !b? c? ( mirror://binhost/ca18d86a2b ) a? b? !c? ( mirror://binhost/6bce13471a ) a? b? c? ( mirror://binhost/3a6bdcd228 ) ) splitdebug? ( !a? !b? !c? ( mirror://binhost/29b2f38c41 ) !a? !b? c? ( mirror://binhost/8adc9bef51 ) !a? b? !c? ( mirror://binhost/954d2ce484 ) !a? b? c? ( mirror://binhost/32a614aaca ) a? !b? !c? ( mirror://binhost/3548a2302d ) a? !b? c? ( mirror://binhost/e0c02cdc88 ) a? b? !c? ( mirror://binhost/f9cbd3c181 ) a? b? c? ( mirror://binhost/31d4c03474 ) ) " For installsources, we can automate deduplication, so that we can distribute the same file content for multiple USE combinations when appropriate. If all of the combinations have identical content, then it will look like this: installsources? ( !
[gentoo-portage-dev] Re: [PATCH] repoman: revert preserve_old_lib deprecation (bug 480244)
On 2/23/21 3:41 PM, Zac Medico wrote: > Repoman should not report that preserve_old_lib is deprecated, > since preserve-libs is not covered by PMS. > This reverts commit 49cbc17bf7b99be586e158c1bd588cfe91dfe58c. > > Bug: https://bugs.gentoo.org/480244 > Bug: https://bugs.gentoo.org/692486 > Signed-off-by: Zac Medico > --- > repoman/cnf/linechecks/linechecks.yaml| 2 +- > .../lib/repoman/modules/linechecks/deprecated/deprecated.py | 4 ++-- > 2 files changed, 3 insertions(+), 3 deletions(-) Due to controversy, I would like to withdraw this patch, and call upon interested parties to help decide an appropriate course of action or inaction for https://bugs.gentoo.org/480244. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] repoman: revert preserve_old_lib deprecation (bug 480244)
Repoman should not report that preserve_old_lib is deprecated, since preserve-libs is not covered by PMS. This reverts commit 49cbc17bf7b99be586e158c1bd588cfe91dfe58c. Bug: https://bugs.gentoo.org/480244 Bug: https://bugs.gentoo.org/692486 Signed-off-by: Zac Medico --- repoman/cnf/linechecks/linechecks.yaml| 2 +- .../lib/repoman/modules/linechecks/deprecated/deprecated.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/repoman/cnf/linechecks/linechecks.yaml b/repoman/cnf/linechecks/linechecks.yaml index 2182b467a..7c040e2b2 100644 --- a/repoman/cnf/linechecks/linechecks.yaml +++ b/repoman/cnf/linechecks/linechecks.yaml @@ -25,7 +25,7 @@ errors: DEPRECATED_BINDNOW_FLAGS: 'Deprecated bindnow-flags call' EAPI_DEFINED_AFTER_INHERIT: 'EAPI defined after inherit' NO_AS_NEEDED: 'Upstream asneeded linking bug (no-as-needed)' -PRESERVE_OLD_LIB: 'Ebuild calls deprecated preserve_old_lib' +PRESERVE_OLD_LIB: 'Upstream ABI change workaround on line: %d' BUILT_WITH_USE: 'built_with_use' NO_OFFSET_WITH_HELPERS: 'Helper function is used with D, ROOT, ED, EROOT or EPREFIX' USEQ_ERROR: 'Ebuild calls deprecated useq function' diff --git a/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py b/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py index d1a590f1d..ad488c53a 100644 --- a/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py +++ b/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py @@ -19,8 +19,8 @@ class DeprecatedHasq(LineCheck): class PreserveOldLib(LineCheck): - """Check for calls to the deprecated preserve_old_lib function.""" - repoman_check_name = 'ebuild.minorsyn' + """Check for calls to the preserve_old_lib function.""" + repoman_check_name = 'upstream.workaround' re = re.compile(r'.*preserve_old_lib') error = 'PRESERVE_OLD_LIB' -- 2.26.2
Re: [gentoo-dev] New project: binhost
On 2/23/21 12:05 PM, Zac Medico wrote: > On 2/23/21 11:46 AM, Zac Medico wrote: >> On 2/20/21 8:17 PM, Zac Medico wrote: >>> IUSE_RUNTIME will obviously introduce conditionals in binary package >>> dependencies, but we should welcome these conditionals because they will >>> provide useful flexibility. >>> >>> I think IUSE_RUNTIME will be a very nice feature to have for project >>> binhost, since it will allow binary package dependencies to behave with >>> flexibility that more closely resembles the flexibility of ebuild >>> dependencies. >> >> We can borrow paludis's notion of pbins [1] to generate ebuilds which >> install pre-built content, and the generated ebuilds could have USE >> flags that behave similarly to IUSE_RUNTIME in the sense that changes to >> USE flags will result in different builds of pre-built content being >> installed. A content-hash distfiles layout [2] could serve as a >> convenient way to store separate builds of pre-built content for >> multiple combinations of USE flags, and a generated ebuild would index >> the build by USE flag combination. >> >> Also, for the generated ebuilds, we can generate USE flags to include >> separate SRC_URI downloads for pre-built content to support things like >> FEATURES=split-debug and FEATURES=install-sources. > > Note that all of this can use existing EAPI features, since everything new > would be implemented in an ebuild generator that generates a single > ebuild to index pre-built content from multiple binary package builds. > >> [1] https://paludis.exherbo.org/overview/pbins.html >> [2] https://bugs.gentoo.org/756778 > For generated ebuilds, we'll have a choice to model things as USE flags or sub-packages. For example, content from FEATURES=split-debug and FEATURES=install-sources content might make more sense to model as sub-packages than USE flags. It makes more sense to generate a sub-package when there is a need for the sub-package to have USE flags. For example, a split-debug sub-package can have USE flags which index pre-built content from builds for multiple USE flag combinations. Similar USE flags could be useful for an install-sources sub-package if the source code has patches which are conditional on USE flags. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-dev] New project: binhost
On 2/23/21 11:46 AM, Zac Medico wrote: > On 2/20/21 8:17 PM, Zac Medico wrote: >> On 2/13/21 4:53 PM, Zac Medico wrote: >>> On 2/13/21 4:37 PM, Zac Medico wrote: >>>> On 2/11/21 1:17 AM, Michał Górny wrote: >>>>> On Wed, 2021-02-10 at 19:51 +0100, Lars Wendler wrote: >>>>>> On Wed, 10 Feb 2021 19:57:48 +0200 Andreas K. Hüttel wrote: >>>>>> >>>>>>> Hi all, >>>>>>> >>>>>>> I'm announcing a new project here - "binhost" >>>>>>> >>>>>>> "The Gentoo Binhost project aims to provide readily installable, >>>>>>> precompiled packages for a subset of configurations, via central >>>>>>> binary package hosting. Currently we are still in the conceptual >>>>>>> planning stage. " >>>>>>> >>>>>>> https://wiki.gentoo.org/wiki/Project:Binhost >>>>>>> >>>>>>> If you're interested in helping out, feel free to add yourself on the >>>>>>> wiki page. >>>>>>> >>>>>>> Note that I see actually *building* the packages not as the central >>>>>>> point of the project (that could be e.g. a side effect of a >>>>>>> tinderbox). I'm more concerned about >>>>>>> * what configurations should we use >>>>>>> * what portage features are still needed or need improvements (e.g. >>>>>>> binpkg signing and verification) >>>>>>> * how should hosting look like >>>>>>> * and how we can test this on a limited scale before it goes "into >>>>>>> production" >>>>>>> * ... >>>>>>> >>>>>>> Comments, ideas, flamebaits? :D >>>>>>> >>>>>>> Cheers, >>>>>>> Andreas >>>>>>> >>>>>> >>>>>> It would be great to improve portage speed with handling binpkgs. I >>>>>> already have my own binhost for a couple of Gentoo systems and even >>>>>> though these systems don't have to compile anything themselves, >>>>>> installing ~100 to ~200 binpkgs takes way more than an hour of >>>>>> installation time. Arch Linux' pacman only takes a fraction of this >>>>>> time for the very same task. >>>>>> I know that I compare apples with pears here but even reducing the >>>>>> current portage time by 50% would be a huge improvement. >>>>> >>>>> Is that really a problem? For me, Portage takes about an hour just to >>>>> do the dependency processing these days. In fact, building from sources >>>>> is now faster than dependency calculations. >>>> >>>> The ratio of these times is dependent on the complexity of the >>>> dependencies involved, and so is the answer to your question. >>> >>> Also, in the context of binary packages, dependencies calculations are >>> generally simpler for binary packages because their USE conditionals and >>> slot-operator := dependencies are frozen in a particular state. This >>> dramatically reduces the search space involved in dependency calculation. >> >> IUSE_RUNTIME will obviously introduce conditionals in binary package >> dependencies, but we should welcome these conditionals because they will >> provide useful flexibility. >> >> I think IUSE_RUNTIME will be a very nice feature to have for project >> binhost, since it will allow binary package dependencies to behave with >> flexibility that more closely resembles the flexibility of ebuild >> dependencies. > > We can borrow paludis's notion of pbins [1] to generate ebuilds which > install pre-built content, and the generated ebuilds could have USE > flags that behave similarly to IUSE_RUNTIME in the sense that changes to > USE flags will result in different builds of pre-built content being > installed. A content-hash distfiles layout [2] could serve as a > convenient way to store separate builds of pre-built content for > multiple combinations of USE flags, and a generated ebuild would index > the build by USE flag combination. > > Also, for the generated ebuilds, we can generate USE flags to include > separate SRC_URI downloads for pre-built content to support things like > FEATURES=split-debug and FEATURES=install-sources. Note that all of this can existing EAPI features, since everything new would be implemented in an ebuild generator that generates a single ebuild to index pre-built content from multiple binary package builds. > [1] https://paludis.exherbo.org/overview/pbins.html > [2] https://bugs.gentoo.org/756778 -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-dev] New project: binhost
On 2/13/21 5:51 PM, Zac Medico wrote: > On 2/10/21 11:11 AM, Rich Freeman wrote: >> On Wed, Feb 10, 2021 at 12:57 PM Andreas K. Hüttel >> wrote: >>> >>> * what portage features are still needed or need improvements (e.g. binpkg >>> signing and verification) >>> * how should hosting look like >> >> Some ideas for portage enhancements: >> >> 1. Ability to fetch binary packages from some kind of repo. > > The old PORTAGE_BINHOST functionality has been replaced with a > binrepos.conf file that's very similar to repos.conf: > > https://bugs.gentoo.org/668334 > > It doesn't have explicit support for multiple local binary package > repositories yet, but somebody got it working with src-uri set to a > file:/ uri as described in comments on this bug: > > https://bugs.gentoo.org/768957 > >> 2. Ability to have multiple binary packages co-exist in a repo (local >> or remote) with different build attributes (arch, USE, CFLAGS, >> DEPENDS, whatever). > > We can now enable FEATURES=binpkg-multi-instance by default now that > this bug is fixed: > > https://bugs.gentoo.org/571126 > >> 3. Ability to pick the most appropriate binary packages to use based >> on user preferences (with a mix of hard and soft preferences). > > Current package selection logic for binary packages is basically the > same as for ebuilds. These are the notable differences: > > 1) Binary packages are sorted in descending order by (version, mtime), > so then most recent builds are preferred when the versions are identical. > > 2) The --binpkg-respect-use option rejects binary packages what would > need to be rebuilt in order to match local USE settings. > >> One idea I've had around how #2-3 might be implemented is: >> 1. Binary packages already contain data on how they were built (USE >> flags, dependencies, etc). Place this in a file using a deterministic >> sorting/etc order so that two builds with the same settings will have >> the same results. > > This would only be needed to multi-profile binhosts that provide a > variety of configurations for the same package. > > Features like this are not necessary if the binhost only intends to > provide packages for a single profile. > >> 2. Generate a hash of the file contents - this can go in the filename >> so that the file can co-exist with other files, and be located >> assuming you have a full matching set of metadata. > > For FEATURES=binpkg-multi-instance we currently use an integer BUILD_ID > ensure that file names are unique. > >> 3. Start dropping attributes from the file based on a list of >> priorities and generate additional hashes. Create symlinked files to >> the original file using these hashes (overwriting or not existing >> symlinks based on policy). This allows the binary package to be found >> using either an exact set of attributes or a subset of higher-priority >> attributes. This is analogous to shared object symlinking. >> 4. The package manager will look for a binary package first using the >> user's full config, and then by dropping optional elements of the >> config (so maybe it does the search without CFLAGs, then without USE >> flags). Eventually it aborts based on user prefs (maybe the user only >> wants an exact match, or is willing to accept alternate CFLAGs but not >> USE flags, or maybe anything for the arch is selected> 5. As always the >> final selected binary package still gets evaluated >> like any other binary package to ensure it is usable. >> >> Such a system can identify whether a potentially usable file exists >> using only filename, cutting down on fetching. In the interests of >> avoiding useless fetches we would only carry step 3 reasonably far - >> packages would have to match based on architecture and any dynamic >> linking requirements. So we wouldn't generate hashes that didn't >> include at least those minimums, and the package manager wouldn't >> search for them. >> >> Obviously you could do more (if you have 5 combinations of use flags, >> look for the set that matches most closely). That couldn't be done >> using hashes alone in an efficient way. You could have a small >> manifest file alongside the binary package that could be fetched >> separately if the package manager wants to narrow things down and >> fetch a few of those to narrow it down further. > > All of the above is oriented toward multi-profile binhosts, so we'll > have to do a cost/benefit analysis to determine whether it's worth the > effort to introduce the complexity that multi-profile binhosts add. This idea to borrow paludis's notion of pbins could work well for a multi-profile binhost: https://archives.gentoo.org/gentoo-dev/message/1aea0de0ff588c67bf652f4c1f8ef304 -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-dev] New project: binhost
On 2/20/21 8:17 PM, Zac Medico wrote: > On 2/13/21 4:53 PM, Zac Medico wrote: >> On 2/13/21 4:37 PM, Zac Medico wrote: >>> On 2/11/21 1:17 AM, Michał Górny wrote: >>>> On Wed, 2021-02-10 at 19:51 +0100, Lars Wendler wrote: >>>>> On Wed, 10 Feb 2021 19:57:48 +0200 Andreas K. Hüttel wrote: >>>>> >>>>>> Hi all, >>>>>> >>>>>> I'm announcing a new project here - "binhost" >>>>>> >>>>>> "The Gentoo Binhost project aims to provide readily installable, >>>>>> precompiled packages for a subset of configurations, via central >>>>>> binary package hosting. Currently we are still in the conceptual >>>>>> planning stage. " >>>>>> >>>>>> https://wiki.gentoo.org/wiki/Project:Binhost >>>>>> >>>>>> If you're interested in helping out, feel free to add yourself on the >>>>>> wiki page. >>>>>> >>>>>> Note that I see actually *building* the packages not as the central >>>>>> point of the project (that could be e.g. a side effect of a >>>>>> tinderbox). I'm more concerned about >>>>>> * what configurations should we use >>>>>> * what portage features are still needed or need improvements (e.g. >>>>>> binpkg signing and verification) >>>>>> * how should hosting look like >>>>>> * and how we can test this on a limited scale before it goes "into >>>>>> production" >>>>>> * ... >>>>>> >>>>>> Comments, ideas, flamebaits? :D >>>>>> >>>>>> Cheers, >>>>>> Andreas >>>>>> >>>>> >>>>> It would be great to improve portage speed with handling binpkgs. I >>>>> already have my own binhost for a couple of Gentoo systems and even >>>>> though these systems don't have to compile anything themselves, >>>>> installing ~100 to ~200 binpkgs takes way more than an hour of >>>>> installation time. Arch Linux' pacman only takes a fraction of this >>>>> time for the very same task. >>>>> I know that I compare apples with pears here but even reducing the >>>>> current portage time by 50% would be a huge improvement. >>>> >>>> Is that really a problem? For me, Portage takes about an hour just to >>>> do the dependency processing these days. In fact, building from sources >>>> is now faster than dependency calculations. >>> >>> The ratio of these times is dependent on the complexity of the >>> dependencies involved, and so is the answer to your question. >> >> Also, in the context of binary packages, dependencies calculations are >> generally simpler for binary packages because their USE conditionals and >> slot-operator := dependencies are frozen in a particular state. This >> dramatically reduces the search space involved in dependency calculation. > > IUSE_RUNTIME will obviously introduce conditionals in binary package > dependencies, but we should welcome these conditionals because they will > provide useful flexibility. > > I think IUSE_RUNTIME will be a very nice feature to have for project > binhost, since it will allow binary package dependencies to behave with > flexibility that more closely resembles the flexibility of ebuild > dependencies. We can borrow paludis's notion of pbins [1] to generate ebuilds which install pre-built content, and the generated ebuilds could have USE flags that behave similarly to IUSE_RUNTIME in the sense that changes to USE flags will result in different builds of pre-built content being installed. A content-hash distfiles layout [2] could serve as a convenient way to store separate builds of pre-built content for multiple combinations of USE flags, and a generated ebuild would index the build by USE flag combination. Also, for the generated ebuilds, we can generate USE flags to include separate SRC_URI downloads for pre-built content to support things like FEATURES=split-debug and FEATURES=install-sources. [1] https://paludis.exherbo.org/overview/pbins.html [2] https://bugs.gentoo.org/756778 -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] Add content-hash distfiles layout (bug 756778)
From: Daniel Robbins
The content-hash layout is identical to the filename-hash layout,
except for these two differences:
1) A content digest is used instead of a filename digest.
2) The final element of the path returned from the get_path method
corresponds to the complete content digest. The path is a
function of the content digest alone.
Motivations to use the content-hash layout instead of the
filename-hash layout may include:
1) Since the file path is independent of the file name, file
name collisions cannot occur. This makes the content-hash
layout suitable for storage of multiple types of files (not
only gentoo distfiles). For example, it can be used to store
distfiles for multiple linux distros within the same tree,
with automatic deduplication based on content digest. This
layout can be used to store and distribute practically anything
(including binary packages for example).
2) Allows multiple revisions for the same distfiles name. An
existing distfile can be updated, and if a user still has an
older copy of an ebuild repository (or an overlay), then a user
can successfully fetch a desired revision of the distfile as
long as it has not been purged from the mirror.
3) File integrity data is integrated into the layout itself,
making it very simple to verify the integrity of any file that
it contains. The only tool required is an implementation of
the chosen hash algorithm.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico
---
lib/portage/package/ebuild/fetch.py| 160 +++--
lib/portage/tests/ebuild/test_fetch.py | 40 ++-
2 files changed, 184 insertions(+), 16 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py
b/lib/portage/package/ebuild/fetch.py
index e0fecaf23..7d2ef93bf 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -344,6 +344,31 @@ _size_suffix_map = {
}
+class DistfileName(str):
+ def __new__(cls, s, digests=None):
+ return str.__new__(cls, s)
+
+ def __init__(self, s, digests=None):
+ super().__init__()
+ self.digests = {} if digests is None else digests
+
+ def digests_equal(self, other):
+ """
+ Test if digests compare equal to those of another instance.
+ """
+ if not isinstance(other, DistfileName):
+ return False
+ matches = []
+ for algo, digest in self.digests.items():
+ other_digest = other.digests.get(algo)
+ if other_digest is not None:
+ if other_digest == digest:
+ matches.append(algo)
+ else:
+ return False
+ return bool(matches)
+
+
class FlatLayout:
def get_path(self, filename):
return filename
@@ -413,6 +438,90 @@ class FilenameHashLayout:
return False
+class ContentHashLayout(FilenameHashLayout):
+ """
+ The content-hash layout is identical to the filename-hash layout,
+ except for these two differences:
+
+ 1) A content digest is used instead of a filename digest.
+ 2) The final element of the path returned from the get_path method
+ corresponds to the complete content digest. The path is a
+ function of the content digest alone.
+
+ Motivations to use the content-hash layout instead of the
+ filename-hash layout may include:
+
+ 1) Since the file path is independent of the file name, file
+ name collisions cannot occur. This makes the content-hash
+ layout suitable for storage of multiple types of files (not
+ only gentoo distfiles). For example, it can be used to store
+ distfiles for multiple linux distros within the same tree,
+ with automatic deduplication based on content digest. This
+ layout can be used to store and distribute practically anything
+ (including binary packages for example).
+
+ 2) Allows multiple revisions for the same distfiles name. An
+ existing distfile can be updated, and if a user still has an
+ older copy of an ebuild repository (or an overlay), then a user
+ can successfully fetch a desired revision of the distfile as
+ long as it has not been purged from the mirror.
+
+ 3) File integrity data is integrated into the layout itself,
+ making it very simple to verify the integrity of any file that
+ it contains. The only tool required is an implementation of
+ the chosen hash algorithm.
+ """
+
+ def get_path(self, filename):
+ """
+ For content-hash, the path is a function of the content digest
alone.
+ The final element of the path re
Re: [gentoo-dev] New project: binhost
On 2/13/21 4:53 PM, Zac Medico wrote: > On 2/13/21 4:37 PM, Zac Medico wrote: >> On 2/11/21 1:17 AM, Michał Górny wrote: >>> On Wed, 2021-02-10 at 19:51 +0100, Lars Wendler wrote: >>>> On Wed, 10 Feb 2021 19:57:48 +0200 Andreas K. Hüttel wrote: >>>> >>>>> Hi all, >>>>> >>>>> I'm announcing a new project here - "binhost" >>>>> >>>>> "The Gentoo Binhost project aims to provide readily installable, >>>>> precompiled packages for a subset of configurations, via central >>>>> binary package hosting. Currently we are still in the conceptual >>>>> planning stage. " >>>>> >>>>> https://wiki.gentoo.org/wiki/Project:Binhost >>>>> >>>>> If you're interested in helping out, feel free to add yourself on the >>>>> wiki page. >>>>> >>>>> Note that I see actually *building* the packages not as the central >>>>> point of the project (that could be e.g. a side effect of a >>>>> tinderbox). I'm more concerned about >>>>> * what configurations should we use >>>>> * what portage features are still needed or need improvements (e.g. >>>>> binpkg signing and verification) >>>>> * how should hosting look like >>>>> * and how we can test this on a limited scale before it goes "into >>>>> production" >>>>> * ... >>>>> >>>>> Comments, ideas, flamebaits? :D >>>>> >>>>> Cheers, >>>>> Andreas >>>>> >>>> >>>> It would be great to improve portage speed with handling binpkgs. I >>>> already have my own binhost for a couple of Gentoo systems and even >>>> though these systems don't have to compile anything themselves, >>>> installing ~100 to ~200 binpkgs takes way more than an hour of >>>> installation time. Arch Linux' pacman only takes a fraction of this >>>> time for the very same task. >>>> I know that I compare apples with pears here but even reducing the >>>> current portage time by 50% would be a huge improvement. >>> >>> Is that really a problem? For me, Portage takes about an hour just to >>> do the dependency processing these days. In fact, building from sources >>> is now faster than dependency calculations. >> >> The ratio of these times is dependent on the complexity of the >> dependencies involved, and so is the answer to your question. > > Also, in the context of binary packages, dependencies calculations are > generally simpler for binary packages because their USE conditionals and > slot-operator := dependencies are frozen in a particular state. This > dramatically reduces the search space involved in dependency calculation. IUSE_RUNTIME will obviously introduce conditionals in binary package dependencies, but we should welcome these conditionals because they will provide useful flexibility. I think IUSE_RUNTIME will be a very nice feature to have for project binhost, since it will allow binary package dependencies to behave with flexibility that more closely resembles the flexibility of ebuild dependencies. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] make.defaults: prevent USE="${USE} ..." misbehavior
Discard parent profile USE from the expand_map variable before
it is used to evaluate a child profile. This prevents accidents
triggered by USE="${USE} ..." settngs at the top of make.defaults
which caused parent profile USE to override parent profile
package.use settings.
Bug: https://bugs.gentoo.org/771549
Signed-off-by: Zac Medico
---
lib/portage/package/ebuild/config.py | 13 ++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/lib/portage/package/ebuild/config.py
b/lib/portage/package/ebuild/config.py
index e5ec681af..638c72959 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -612,9 +612,16 @@ class config:
mygcfg = {}
if profiles_complex:
- mygcfg_dlists =
[getconfig(os.path.join(x.location, "make.defaults"),
- tolerant=tolerant, expand=expand_map,
recursive=x.portage1_directories)
- for x in profiles_complex]
+ mygcfg_dlists = []
+ for x in profiles_complex:
+ # Prevent accidents triggered by
USE="${USE} ..." settings
+ # at the top of make.defaults which
caused parent profile
+ # USE to override parent profile
package.use settings.
+ expand_map.pop("USE", None)
+ mygcfg_dlists.append(
+
getconfig(os.path.join(x.location, "make.defaults"),
+ tolerant=tolerant,
expand=expand_map,
+
recursive=x.portage1_directories))
self._make_defaults = mygcfg_dlists
mygcfg = stack_dicts(mygcfg_dlists,
incrementals=self.incrementals)
--
2.26.2
Re: [gentoo-dev] New project: binhost
On 2/10/21 11:11 AM, Rich Freeman wrote: > On Wed, Feb 10, 2021 at 12:57 PM Andreas K. Hüttel > wrote: >> >> * what portage features are still needed or need improvements (e.g. binpkg >> signing and verification) >> * how should hosting look like > > Some ideas for portage enhancements: > > 1. Ability to fetch binary packages from some kind of repo. The old PORTAGE_BINHOST functionality has been replaced with a binrepos.conf file that's very similar to repos.conf: https://bugs.gentoo.org/668334 It doesn't have explicit support for multiple local binary package repositories yet, but somebody got it working with src-uri set to a file:/ uri as described in comments on this bug: https://bugs.gentoo.org/768957 > 2. Ability to have multiple binary packages co-exist in a repo (local > or remote) with different build attributes (arch, USE, CFLAGS, > DEPENDS, whatever). We can now enable FEATURES=binpkg-multi-instance by default now that this bug is fixed: https://bugs.gentoo.org/571126 > 3. Ability to pick the most appropriate binary packages to use based > on user preferences (with a mix of hard and soft preferences). Current package selection logic for binary packages is basically the same as for ebuilds. These are the notable differences: 1) Binary packages are sorted in descending order by (version, mtime), so then most recent builds are preferred when the versions are identical. 2) The --binpkg-respect-use option rejects binary packages what would need to be rebuilt in order to match local USE settings. > One idea I've had around how #2-3 might be implemented is: > 1. Binary packages already contain data on how they were built (USE > flags, dependencies, etc). Place this in a file using a deterministic > sorting/etc order so that two builds with the same settings will have > the same results. This would only be needed to multi-profile binhosts that provide a variety of configurations for the same package. Features like this are not necessary if the binhost only intends to provide packages for a single profile. > 2. Generate a hash of the file contents - this can go in the filename > so that the file can co-exist with other files, and be located > assuming you have a full matching set of metadata. For FEATURES=binpkg-multi-instance we currently use an integer BUILD_ID ensure that file names are unique. > 3. Start dropping attributes from the file based on a list of > priorities and generate additional hashes. Create symlinked files to > the original file using these hashes (overwriting or not existing > symlinks based on policy). This allows the binary package to be found > using either an exact set of attributes or a subset of higher-priority > attributes. This is analogous to shared object symlinking. > 4. The package manager will look for a binary package first using the > user's full config, and then by dropping optional elements of the > config (so maybe it does the search without CFLAGs, then without USE > flags). Eventually it aborts based on user prefs (maybe the user only > wants an exact match, or is willing to accept alternate CFLAGs but not > USE flags, or maybe anything for the arch is selected> 5. As always the > final selected binary package still gets evaluated > like any other binary package to ensure it is usable. > > Such a system can identify whether a potentially usable file exists > using only filename, cutting down on fetching. In the interests of > avoiding useless fetches we would only carry step 3 reasonably far - > packages would have to match based on architecture and any dynamic > linking requirements. So we wouldn't generate hashes that didn't > include at least those minimums, and the package manager wouldn't > search for them. > > Obviously you could do more (if you have 5 combinations of use flags, > look for the set that matches most closely). That couldn't be done > using hashes alone in an efficient way. You could have a small > manifest file alongside the binary package that could be fetched > separately if the package manager wants to narrow things down and > fetch a few of those to narrow it down further. All of the above is oriented toward multi-profile binhosts, so we'll have to do a cost/benefit analysis to determine whether it's worth the effort to introduce the complexity that multi-profile binhosts add. > Or you could skip the hash searching and just fetch all the manifests > for a particular package/arch and just search all of those, but that > is more data to transfer just to do a query. A metadata cache of some > kind of might be another solution. Content hashes would probably > still be useful just to allow co-existence of alternate builds. This also relates to the centralized Packages file that's currently used to distribute the package metadata for all packages in a binhost. We can make it scale better if we split out a separate index per package, not unlike a pypi simple index: https://pypi.org/simple/ --
Re: [gentoo-dev] New project: binhost
On 2/13/21 4:37 PM, Zac Medico wrote: > On 2/11/21 1:17 AM, Michał Górny wrote: >> On Wed, 2021-02-10 at 19:51 +0100, Lars Wendler wrote: >>> On Wed, 10 Feb 2021 19:57:48 +0200 Andreas K. Hüttel wrote: >>> >>>> Hi all, >>>> >>>> I'm announcing a new project here - "binhost" >>>> >>>> "The Gentoo Binhost project aims to provide readily installable, >>>> precompiled packages for a subset of configurations, via central >>>> binary package hosting. Currently we are still in the conceptual >>>> planning stage. " >>>> >>>> https://wiki.gentoo.org/wiki/Project:Binhost >>>> >>>> If you're interested in helping out, feel free to add yourself on the >>>> wiki page. >>>> >>>> Note that I see actually *building* the packages not as the central >>>> point of the project (that could be e.g. a side effect of a >>>> tinderbox). I'm more concerned about >>>> * what configurations should we use >>>> * what portage features are still needed or need improvements (e.g. >>>> binpkg signing and verification) >>>> * how should hosting look like >>>> * and how we can test this on a limited scale before it goes "into >>>> production" >>>> * ... >>>> >>>> Comments, ideas, flamebaits? :D >>>> >>>> Cheers, >>>> Andreas >>>> >>> >>> It would be great to improve portage speed with handling binpkgs. I >>> already have my own binhost for a couple of Gentoo systems and even >>> though these systems don't have to compile anything themselves, >>> installing ~100 to ~200 binpkgs takes way more than an hour of >>> installation time. Arch Linux' pacman only takes a fraction of this >>> time for the very same task. >>> I know that I compare apples with pears here but even reducing the >>> current portage time by 50% would be a huge improvement. >> >> Is that really a problem? For me, Portage takes about an hour just to >> do the dependency processing these days. In fact, building from sources >> is now faster than dependency calculations. > > The ratio of these times is dependent on the complexity of the > dependencies involved, and so is the answer to your question. Also, in the context of binary packages, dependencies calculations are generally simpler for binary packages because their USE conditionals and slot-operator := dependencies are frozen in a particular state. This dramatically reduces the search space involved in dependency calculation. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-dev] New project: binhost
On 2/11/21 1:17 AM, Michał Górny wrote: > On Wed, 2021-02-10 at 19:51 +0100, Lars Wendler wrote: >> On Wed, 10 Feb 2021 19:57:48 +0200 Andreas K. Hüttel wrote: >> >>> Hi all, >>> >>> I'm announcing a new project here - "binhost" >>> >>> "The Gentoo Binhost project aims to provide readily installable, >>> precompiled packages for a subset of configurations, via central >>> binary package hosting. Currently we are still in the conceptual >>> planning stage. " >>> >>> https://wiki.gentoo.org/wiki/Project:Binhost >>> >>> If you're interested in helping out, feel free to add yourself on the >>> wiki page. >>> >>> Note that I see actually *building* the packages not as the central >>> point of the project (that could be e.g. a side effect of a >>> tinderbox). I'm more concerned about >>> * what configurations should we use >>> * what portage features are still needed or need improvements (e.g. >>> binpkg signing and verification) >>> * how should hosting look like >>> * and how we can test this on a limited scale before it goes "into >>> production" >>> * ... >>> >>> Comments, ideas, flamebaits? :D >>> >>> Cheers, >>> Andreas >>> >> >> It would be great to improve portage speed with handling binpkgs. I >> already have my own binhost for a couple of Gentoo systems and even >> though these systems don't have to compile anything themselves, >> installing ~100 to ~200 binpkgs takes way more than an hour of >> installation time. Arch Linux' pacman only takes a fraction of this >> time for the very same task. >> I know that I compare apples with pears here but even reducing the >> current portage time by 50% would be a huge improvement. > > Is that really a problem? For me, Portage takes about an hour just to > do the dependency processing these days. In fact, building from sources > is now faster than dependency calculations. The ratio of these times is dependent on the complexity of the dependencies involved, and so is the answer to your question. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-dev] New project: binhost
On 2/10/21 10:51 AM, Lars Wendler wrote:
> On Wed, 10 Feb 2021 19:57:48 +0200 Andreas K. Hüttel wrote:
>
>> Hi all,
>>
>> I'm announcing a new project here - "binhost"
>>
>> "The Gentoo Binhost project aims to provide readily installable,
>> precompiled packages for a subset of configurations, via central
>> binary package hosting. Currently we are still in the conceptual
>> planning stage. "
>>
>> https://wiki.gentoo.org/wiki/Project:Binhost
>>
>> If you're interested in helping out, feel free to add yourself on the
>> wiki page.
>>
>> Note that I see actually *building* the packages not as the central
>> point of the project (that could be e.g. a side effect of a
>> tinderbox). I'm more concerned about
>> * what configurations should we use
>> * what portage features are still needed or need improvements (e.g.
>> binpkg signing and verification)
>> * how should hosting look like
>> * and how we can test this on a limited scale before it goes "into
>> production"
>> * ...
>>
>> Comments, ideas, flamebaits? :D
>>
>> Cheers,
>> Andreas
>>
>
> It would be great to improve portage speed with handling binpkgs. I
> already have my own binhost for a couple of Gentoo systems and even
> though these systems don't have to compile anything themselves,
> installing ~100 to ~200 binpkgs takes way more than an hour of
> installation time. Arch Linux' pacman only takes a fraction of this
> time for the very same task.
> I know that I compare apples with pears here but even reducing the
> current portage time by 50% would be a huge improvement.
In order to maximize throughput, we have FEATURES="parallel-install"
that for example allows one package's files to be merged while a
pkg_{pre,post}{inst,rm} phase is running for an unrelated package that
is in the merge queue.
There's also FEATURES="-ebuild-locks" which allows privileged
pkg_{pre,post}{inst,rm} phases to run concurrently with privileged
phases of unrelated packages in the merge queue.
Ultimately, pkg_{pre,post}{inst,rm} phases could be the limiting factor
here. I portage, we should eliminate calls to these phases when
DEFINED_PHASES metadata indicated that they're not defined.
Also, FEATURES=preserve-libs introduces considerable overhead because it
regenerates LinkageMapELF data structures for every merge. It would be
much more efficient if it could incrementally update these data structures.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] portage.getpid: call os.getpid() lazily
Call os.getpid() lazily, which eliminates getpid calls when possible after os.fork() in the portage.process module. Bug: https://bugs.gentoo.org/767913 Signed-off-by: Zac Medico --- lib/portage/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py index 3c9f78497..24c9d8b89 100644 --- a/lib/portage/__init__.py +++ b/lib/portage/__init__.py @@ -375,7 +375,7 @@ _sync_mode = False class _ForkWatcher: @staticmethod def hook(_ForkWatcher): - _ForkWatcher.current_pid = _os.getpid() + _ForkWatcher.current_pid = None # Force instantiation of a new event loop policy as a workaround # for https://bugs.python.org/issue22087. asyncio.set_event_loop_policy(None) @@ -388,6 +388,8 @@ def getpid(): """ Cached version of os.getpid(). ForkProcess updates the cache. """ + if _ForkWatcher.current_pid is None: + _ForkWatcher.current_pid = _os.getpid() return _ForkWatcher.current_pid def _get_stdin(): -- 2.26.2
[gentoo-portage-dev] [PATCH] emerge: disable --autounmask-license by default
Disable --autounmask-license by default, in order to limit user
exposure to risks associated with package.license changes.
The changes that this option suggests are only intended to be
accepted when a user has made a conscious decision to accept
the corresponding license(s). Creation of package.license
changes introduces a risk that users may erroneously accept the
changes due to some kind of accident or misunderstanding,
rather than due to conscious decisions about licenses.
These risks provide motivation to disable --autounmask-license
by default. The --autounmask-use option will remain as the
only autounmask option that is still enabled by default.
The unit tests demonstrate interactions between --autounmask
and --autounmask-license options. The --autounmask option
enables --autounmask-license unless --autounmask-license=n
has been specified. If --autounmask=n is used to disable
autounmask, then --autounmask-license=y has no effect.
Bug: https://bugs.gentoo.org/766773
Signed-off-by: Zac Medico
---
lib/_emerge/create_depgraph_params.py | 8 +++---
lib/portage/tests/resolver/test_autounmask.py | 25 +--
man/emerge.1 | 11 +++-
3 files changed, 31 insertions(+), 13 deletions(-)
diff --git a/lib/_emerge/create_depgraph_params.py
b/lib/_emerge/create_depgraph_params.py
index 0d0e07b9c..25dd2a1b4 100644
--- a/lib/_emerge/create_depgraph_params.py
+++ b/lib/_emerge/create_depgraph_params.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import logging
@@ -45,7 +45,7 @@ def create_depgraph_params(myopts, myaction):
autounmask_keep_masks = myopts.get("--autounmask-keep-masks")
autounmask = myopts.get("--autounmask")
- autounmask_license = myopts.get('--autounmask-license')
+ autounmask_license = myopts.get('--autounmask-license', 'y' if
autounmask is True else 'n')
autounmask_use = myopts.get('--autounmask-use')
if autounmask == 'n':
autounmask = False
@@ -53,7 +53,7 @@ def create_depgraph_params(myopts, myaction):
if autounmask is None:
if autounmask_use in (None, 'y'):
autounmask = True
- elif autounmask_license in (None, 'y'):
+ if autounmask_license in ('y',):
autounmask = True
# Do not enable package.accept_keywords or package.mask
@@ -67,7 +67,7 @@ def create_depgraph_params(myopts, myaction):
myparams['autounmask'] = autounmask
myparams['autounmask_keep_use'] = True if autounmask_use == 'n' else
False
- myparams['autounmask_keep_license'] = True if autounmask_license == 'n'
else False
+ myparams['autounmask_keep_license'] = False if autounmask_license ==
'y' else True
myparams['autounmask_keep_keywords'] = False if
autounmask_keep_keywords in (None, 'n') else True
myparams['autounmask_keep_masks'] = False if autounmask_keep_masks in
(None, 'n') else True
diff --git a/lib/portage/tests/resolver/test_autounmask.py
b/lib/portage/tests/resolver/test_autounmask.py
index a3bf0ff94..86ae4bbf6 100644
--- a/lib/portage/tests/resolver/test_autounmask.py
+++ b/lib/portage/tests/resolver/test_autounmask.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2019 Gentoo Authors
+# Copyright 2010-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -440,13 +440,34 @@ class AutounmaskTestCase(TestCase):
mergelist=["dev-libs/A-1"],
license_changes={ "dev-libs/A-1":
set(["TEST"]) }),
- # Test default --autounmask-license
+ # Test that --autounmask enables
--autounmask-license
ResolverPlaygroundTestCase(
["=dev-libs/A-1"],
+ options={"--autounmask": True},
success=False,
mergelist=["dev-libs/A-1"],
license_changes={ "dev-libs/A-1":
set(["TEST"]) }),
+ # Test that --autounmask-license is not enabled
by default
+ ResolverPlaygroundTestCase(
+ ["=dev-libs/A-1"],
+ success=False,
+ ),
+
+ # Test that --autounmask does not override
--autounmask-license=n
+ ResolverPlaygroundTestCase(
+
[gentoo-portage-dev] [PATCH] emaint --fix merges: add -y, --yes option
Since the emaint --fix merges uses emerge --ask, add
a -y, --yes option to use --ask=n instead.
Bug: https://bugs.gentoo.org/766767
Signed-off-by: Zac Medico
---
lib/portage/emaint/modules/merges/__init__.py | 14 --
lib/portage/emaint/modules/merges/merges.py | 11 +++
man/emaint.1 | 6 +-
3 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/lib/portage/emaint/modules/merges/__init__.py
b/lib/portage/emaint/modules/merges/__init__.py
index 89aa758a0..449f39dce 100644
--- a/lib/portage/emaint/modules/merges/__init__.py
+++ b/lib/portage/emaint/modules/merges/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2014 Gentoo Foundation
+# Copyright 2005-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
doc = """Scan for failed merges and fix them."""
@@ -26,7 +26,17 @@ module_spec = {
'action': 'store_true',
'func': 'purge'
}
- }
+ },
+ 'opt_desc': {
+ 'yes': {
+ "short": "-y",
+ "long": "--yes",
+ "help": ("(merges submodule only): Do
not prompt for "
+ "emerge invocations"),
+ "action": "store_true",
+ "dest": "yes",
+ }
+ },
}
}
}
diff --git a/lib/portage/emaint/modules/merges/merges.py
b/lib/portage/emaint/modules/merges/merges.py
index 775dc59d2..d60916f1e 100644
--- a/lib/portage/emaint/modules/merges/merges.py
+++ b/lib/portage/emaint/modules/merges/merges.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2020 Gentoo Authors
+# Copyright 2005-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import portage
@@ -186,7 +186,7 @@ class MergesHandler:
pkg_atoms.add(pkg_atom)
- def _emerge_pkg_atoms(self, module_output, pkg_atoms):
+ def _emerge_pkg_atoms(self, module_output, pkg_atoms, yes=False):
"""
Emerge the specified packages atoms.
@@ -194,6 +194,8 @@ class MergesHandler:
@type module_output: Class
@param pkg_atoms: packages atoms to emerge
@type pkg_atoms: set
+ @param yes: do not prompt for emerge invocations
+ @type yes: bool
@rtype: list
@return: List of results
"""
@@ -206,7 +208,7 @@ class MergesHandler:
portage._python_interpreter,
'-b',
os.path.join(EPREFIX or '/', 'usr', 'bin', 'emerge'),
- '--ask',
+ '--ask=n' if yes else '--ask',
'--quiet',
'--oneshot',
'--complete-graph=y'
@@ -265,7 +267,8 @@ class MergesHandler:
errors.append(', '.join(sorted(failed_pkgs)))
return (False, errors)
self._remove_failed_dirs(failed_pkgs)
- results = self._emerge_pkg_atoms(module_output, pkg_atoms)
+ results = self._emerge_pkg_atoms(module_output, pkg_atoms,
+ yes=kwargs.get('options', {}).get("yes", False))
# list any new failed merges
for pkg in sorted(self._scan()):
results.append("'%s' still found as a failed merge." %
pkg)
diff --git a/man/emaint.1 b/man/emaint.1
index d244756e9..c9f8ab939 100644
--- a/man/emaint.1
+++ b/man/emaint.1
@@ -1,4 +1,4 @@
-.TH "EMAINT" "1" "Jan 2017" "Portage VERSION" "Portage"
+.TH "EMAINT" "1" "Jan 2021" "Portage VERSION" "Portage"
.SH NAME
emaint \- performs package management related system health checks and
maintenance
.SH SYNOPSIS
@@ -82,6 +82,10 @@ OPTION
.BR \-t \ \fINUM\fR,\ \-\-time \ \fINUM\fR
Changes the minimum age \fINUM\fR (in days) of the logs to be listed or
deleted.
+.SH OPTIONS merges command only
+.TP
+.BR \-y ", " \-\-yes
+Do not prompt for emerge invocations.
.SH OPTIONS sync command only
.TP
.BR \-a ", " \-\-auto
--
2.26.2
[gentoo-portage-dev] [PATCH] binarytree.move_ent: copy on write for package move
Copy on write when applying package moves, and silently
skip package moves when the same move has already been
applied to the same build of the package. Since the old
package instance is preserved, it avoids the problem
of having enries for deleted packages remain in the
package index. We can simply assume that the package
will be deleted by eclean-pkg when its time comes.
Bug: https://bugs.gentoo.org/766012
Signed-off-by: Zac Medico
---
lib/portage/dbapi/bintree.py | 40 ---
lib/portage/emaint/modules/move/move.py | 13 ++--
lib/portage/tests/update/test_move_ent.py | 7 ++--
3 files changed, 44 insertions(+), 16 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 180e48c3b..76fca5523 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -31,6 +31,7 @@ from portage.exception import AlarmSignal,
InvalidPackageName, \
ParseError, PortageException
from portage.localization import _
from portage.package.ebuild.profile_iuse import iter_iuse_vars
+from portage.util.file_copy import copyfile
from portage.util.futures import asyncio
from portage.util.futures.compat_coroutine import coroutine
from portage.util.futures.executor.fork import ForkExecutor
@@ -483,6 +484,17 @@ class binarytree:
myoldpkg = catsplit(mycpv)[1]
mynewpkg = catsplit(mynewcpv)[1]
+ # If this update has already been applied to the same
+ # package build then silently continue.
+ applied = False
+ for maybe_applied in
self.dbapi.match('={}'.format(mynewcpv)):
+ if maybe_applied.build_time == mycpv.build_time:
+ applied = True
+ break
+
+ if applied:
+ continue
+
if (mynewpkg != myoldpkg) and
self.dbapi.cpv_exists(mynewcpv):
writemsg(_("!!! Cannot update binary:
Destination exists.\n"),
noiselevel=-1)
@@ -513,24 +525,30 @@ class binarytree:
mydata[_unicode_encode(mynewpkg +
'.ebuild',
encoding=_encodings['repo.content'])] = ebuild_data
- mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
-
- self.dbapi.cpv_remove(mycpv)
- del self._pkg_paths[self.dbapi._instance_key(mycpv)]
metadata = self.dbapi._aux_cache_slot_dict()
for k in self.dbapi._aux_cache_keys:
v = mydata.get(_unicode_encode(k))
if v is not None:
v = _unicode_decode(v)
metadata[k] = " ".join(v.split())
+
+ # Create a copy of the old version of the package and
+ # apply the update to it. Leave behind the old version,
+ # assuming that it will be deleted by eclean-pkg when
its
+ # time comes.
mynewcpv = _pkg_str(mynewcpv, metadata=metadata,
db=self.dbapi)
- new_path = self.getname(mynewcpv)
- self._pkg_paths[
- self.dbapi._instance_key(mynewcpv)] =
new_path[len(self.pkgdir)+1:]
- if new_path != tbz2path:
- self._ensure_dir(os.path.dirname(new_path))
- _movefile(tbz2path, new_path,
mysettings=self.settings)
- self.inject(mynewcpv)
+ update_path = self.getname(mynewcpv, allocate_new=True)
+ ".partial"
+ self._ensure_dir(os.path.dirname(update_path))
+ update_path_lock = None
+ try:
+ update_path_lock = lockfile(update_path,
wantnewlockfile=True)
+ copyfile(tbz2path, update_path)
+ mytbz2 = portage.xpak.tbz2(update_path)
+
mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
+ self.inject(mynewcpv, filename=update_path)
+ finally:
+ if update_path_lock is not None:
+ unlockfile(update_path_lock)
return moves
diff --git a/lib/portage/emaint/modules/move/move.py
b/lib/portage/emaint/modules/move/move.py
index 8fc3269ca..2a95e99c4 100644
--- a/lib/portage/emaint/modules/move/move.py
+++ b/lib/portage/emaint/modules/move/move.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2020 Gentoo
Re: [gentoo-portage-dev] [PATCH] Add @changed-subslot package set
On 1/18/21 8:42 PM, Alec Warner wrote: > On Mon, Jan 18, 2021 at 8:09 PM Zac Medico wrote: >> >> On 1/18/21 6:07 PM, Alec Warner wrote: >>> On Fri, Jan 15, 2021 at 6:47 PM Matt Turner wrote: >>>> >>>> This set is the upgradable packages for which the highest visible >>>> version has a different subslot than the currently installed version. >>>> >>>> The primary purpose of this feature is for use in catalyst builds. We >>>> update the "seed" stage3 before using it to build a new stage1. >>>> >>>> Updating the entire stage is expensive and unnecessary (since we're >>>> going to build the latest packages in stage1 and then rebuild everything >>>> in stage3). >>>> >>>> What we definitely do need to update in the original stage3 however, is >>>> any package that would trigger a subslot rebuild. >>>> >>>> For example: gcc links with libmpfr.so from dev-libs/mpfr. mpfr's SONAME >>>> changes from libmpfr.so.4 (SLOT="0/4") to libmpfr.so.6 (SLOT="0/6"). If >>>> the seed stage's dev-libs/mpfr is not updated before emerging gcc, gcc >>>> will link with libmpfr.so.4, but the latest version of dev-libs/mpfr >>>> will be built and libmpfr.so.6 included into the stage1. Since the old >>>> libmpfr.so.4 is not included in the stage1, gcc will not work, breaking >>>> subsequent stage builds. >>>> >>>> Our current options to update the seed are too large a hammer (e.g., >>>> "--update --deep --newuse @world" or "--update --deep --newuse >>>> --complete-graph --rebuild-if-new-ver gcc") and spend too much time >>>> updating seed stages for no gain beyond updating only packages for whom >>>> the subslot has changed. >>>> >>>> With this set, catalyst will likely use >>>> >>>> emerge @changed-subslot --ignore-built-slot-operator-deps y >>>> >>>> to update the seed stage. >>>> >>>> Thank you to Zac Medico for showing me how to do this. >>>> >>>> Bug: https://bugs.gentoo.org/739004 >>>> Signed-off-by: Matt Turner >>>> --- >>>> cnf/sets/portage.conf | 5 + >>>> lib/portage/_sets/dbapi.py | 39 +- >>>> 2 files changed, 43 insertions(+), 1 deletion(-) >>>> >>>> diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf >>>> index 22f0fa3a5..5651a9c53 100644 >>>> --- a/cnf/sets/portage.conf >>>> +++ b/cnf/sets/portage.conf >>>> @@ -84,6 +84,11 @@ exclude-files = /usr/bin/Xorg >>>> [rebuilt-binaries] >>>> class = portage.sets.dbapi.RebuiltBinaries >>>> >>>> +# Installed packages for which the subslot of the highest visible ebuild >>>> +# version is different than the currently installed version. >>>> +[changed-subslot] >>>> +class = portage.sets.dbapi.SubslotChangedSet >>>> + >>>> # Installed packages for which the highest visible ebuild >>>> # version is lower than the currently installed version. >>>> [downgrade] >>>> diff --git a/lib/portage/_sets/dbapi.py b/lib/portage/_sets/dbapi.py >>>> index 52367c4a6..46ba5c17d 100644 >>>> --- a/lib/portage/_sets/dbapi.py >>>> +++ b/lib/portage/_sets/dbapi.py >>>> @@ -15,7 +15,7 @@ from portage._sets import SetConfigError, get_boolean >>>> import portage >>>> >>>> __all__ = ["CategorySet", "ChangedDepsSet", "DowngradeSet", >>>> - "EverythingSet", "OwnerSet", "VariableSet"] >>>> + "EverythingSet", "OwnerSet", "SubslotChangedSet", "VariableSet"] >>>> >>>> class EverythingSet(PackageSet): >>>> _operations = ["merge"] >>>> @@ -167,6 +167,43 @@ class VariableSet(EverythingSet): >>>> >>>> singleBuilder = classmethod(singleBuilder) >>>> >>>> +class SubslotChangedSet(PackageSet): >>>> + >>>> + _operations = ["merge", "unmerge"] >>>> + >>>> + description = "Package set which contains all packages " + \ >>>> + "for which the subslot of the highest visible ebuild is " >>>> + \ >&g
Re: [gentoo-portage-dev] [PATCH] Add @changed-subslot package set
On 1/18/21 6:07 PM, Alec Warner wrote:
> On Fri, Jan 15, 2021 at 6:47 PM Matt Turner wrote:
>>
>> This set is the upgradable packages for which the highest visible
>> version has a different subslot than the currently installed version.
>>
>> The primary purpose of this feature is for use in catalyst builds. We
>> update the "seed" stage3 before using it to build a new stage1.
>>
>> Updating the entire stage is expensive and unnecessary (since we're
>> going to build the latest packages in stage1 and then rebuild everything
>> in stage3).
>>
>> What we definitely do need to update in the original stage3 however, is
>> any package that would trigger a subslot rebuild.
>>
>> For example: gcc links with libmpfr.so from dev-libs/mpfr. mpfr's SONAME
>> changes from libmpfr.so.4 (SLOT="0/4") to libmpfr.so.6 (SLOT="0/6"). If
>> the seed stage's dev-libs/mpfr is not updated before emerging gcc, gcc
>> will link with libmpfr.so.4, but the latest version of dev-libs/mpfr
>> will be built and libmpfr.so.6 included into the stage1. Since the old
>> libmpfr.so.4 is not included in the stage1, gcc will not work, breaking
>> subsequent stage builds.
>>
>> Our current options to update the seed are too large a hammer (e.g.,
>> "--update --deep --newuse @world" or "--update --deep --newuse
>> --complete-graph --rebuild-if-new-ver gcc") and spend too much time
>> updating seed stages for no gain beyond updating only packages for whom
>> the subslot has changed.
>>
>> With this set, catalyst will likely use
>>
>> emerge @changed-subslot --ignore-built-slot-operator-deps y
>>
>> to update the seed stage.
>>
>> Thank you to Zac Medico for showing me how to do this.
>>
>> Bug: https://bugs.gentoo.org/739004
>> Signed-off-by: Matt Turner
>> ---
>> cnf/sets/portage.conf | 5 +
>> lib/portage/_sets/dbapi.py | 39 +-
>> 2 files changed, 43 insertions(+), 1 deletion(-)
>>
>> diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf
>> index 22f0fa3a5..5651a9c53 100644
>> --- a/cnf/sets/portage.conf
>> +++ b/cnf/sets/portage.conf
>> @@ -84,6 +84,11 @@ exclude-files = /usr/bin/Xorg
>> [rebuilt-binaries]
>> class = portage.sets.dbapi.RebuiltBinaries
>>
>> +# Installed packages for which the subslot of the highest visible ebuild
>> +# version is different than the currently installed version.
>> +[changed-subslot]
>> +class = portage.sets.dbapi.SubslotChangedSet
>> +
>> # Installed packages for which the highest visible ebuild
>> # version is lower than the currently installed version.
>> [downgrade]
>> diff --git a/lib/portage/_sets/dbapi.py b/lib/portage/_sets/dbapi.py
>> index 52367c4a6..46ba5c17d 100644
>> --- a/lib/portage/_sets/dbapi.py
>> +++ b/lib/portage/_sets/dbapi.py
>> @@ -15,7 +15,7 @@ from portage._sets import SetConfigError, get_boolean
>> import portage
>>
>> __all__ = ["CategorySet", "ChangedDepsSet", "DowngradeSet",
>> - "EverythingSet", "OwnerSet", "VariableSet"]
>> + "EverythingSet", "OwnerSet", "SubslotChangedSet", "VariableSet"]
>>
>> class EverythingSet(PackageSet):
>> _operations = ["merge"]
>> @@ -167,6 +167,43 @@ class VariableSet(EverythingSet):
>>
>> singleBuilder = classmethod(singleBuilder)
>>
>> +class SubslotChangedSet(PackageSet):
>> +
>> + _operations = ["merge", "unmerge"]
>> +
>> + description = "Package set which contains all packages " + \
>> + "for which the subslot of the highest visible ebuild is " + \
>> + "different than the currently installed version."
>
> description = ("string1",
>"string2",
>"string3")
>
> vs concat + \ for line continuation?
>
> -A
We also got flak on irc about the classmethod(singleBuilder) usage as
opposed to @classmethod. This package set is formatted exactly like
others in the file, so it's just a copy / paste thing.
On the topic of python formatting, maybe we should use something like
https://github.com/psf/black to automate it?
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH gentoolkit] equery: Remove 'changes' subcommand
On 1/4/21 10:44 AM, Matt Turner wrote: > ChangeLogs have been gone from gentoo.git since the beginning, and > Council agreed in 2016 to allow Infra to decide whether to distribute > them through rsync, which they have decided not to do [1]. > > [1] https://projects.gentoo.org/council/meeting-logs/20160410-summary.txt > > Signed-off-by: Matt Turner > --- > pym/gentoolkit/equery/__init__.py | 1 - > pym/gentoolkit/equery/changes.py| 184 > pym/gentoolkit/helpers.py | 173 -- > pym/gentoolkit/test/equery/test_init.py | 1 - > pym/gentoolkit/test/test_helpers.py | 47 -- > 5 files changed, 406 deletions(-) > delete mode 100644 pym/gentoolkit/equery/changes.py Looks good. We should also be able to remove the egencache --update-changelogs option. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] Re: [PATCH] global_event_loop: return running loop for current thread
On 1/4/21 1:06 AM, Zac Medico wrote: > Like asyncio.get_event_loop(), return the running loop for the > current thread if there is one, and otherwise construct a new > one if needed. This allows the _safe_loop function to become > synonymous with the global_event_loop function. > > Bug: https://bugs.gentoo.org/763339 > Signed-off-by: Zac Medico > --- > .../util/_eventloop/global_event_loop.py | 28 ++- > lib/portage/util/futures/_asyncio/__init__.py | 22 ++- > 2 files changed, 17 insertions(+), 33 deletions(-) For the case of "loop running in non-main thread" of API consumer, this change makes portage compatible with PEP 492 coroutines with async and await syntax. Portage internals can safely begin using async / await syntax instead of compat_coroutine. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] global_event_loop: return running loop for current thread
Like asyncio.get_event_loop(), return the running loop for the
current thread if there is one, and otherwise construct a new
one if needed. This allows the _safe_loop function to become
synonymous with the global_event_loop function.
Bug: https://bugs.gentoo.org/763339
Signed-off-by: Zac Medico
---
.../util/_eventloop/global_event_loop.py | 28 ++-
lib/portage/util/futures/_asyncio/__init__.py | 22 ++-
2 files changed, 17 insertions(+), 33 deletions(-)
diff --git a/lib/portage/util/_eventloop/global_event_loop.py
b/lib/portage/util/_eventloop/global_event_loop.py
index 413011178..bf314dc34 100644
--- a/lib/portage/util/_eventloop/global_event_loop.py
+++ b/lib/portage/util/_eventloop/global_event_loop.py
@@ -1,28 +1,4 @@
-# Copyright 2012-2020 Gentoo Authors
+# Copyright 2012-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import portage
-from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop
-
-_instances = {}
-
-
-def global_event_loop():
- """
- Get a global EventLoop (or compatible object) instance which
- belongs exclusively to the current process.
- """
-
- pid = portage.getpid()
- instance = _instances.get(pid)
- if instance is not None:
- return instance
-
- constructor = AsyncioEventLoop
-
- # Use the _asyncio_wrapper attribute, so that unit tests can compare
- # the reference to one retured from _wrap_loop(), since they should
- # not close the loop if it refers to a global event loop.
- instance = constructor()._asyncio_wrapper
- _instances[pid] = instance
- return instance
+from portage.util.futures._asyncio import _safe_loop as global_event_loop
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index d39f31786..ab1468d43 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018-2020 Gentoo Authors
+# Copyright 2018-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = (
@@ -37,9 +37,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures:compat_coroutine@_compat_coroutine',
)
from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as
_AsyncioEventLoop
-from portage.util._eventloop.global_event_loop import (
- global_event_loop as _global_event_loop,
-)
# pylint: disable=redefined-builtin
from portage.util.futures.futures import (
CancelledError,
@@ -238,7 +235,7 @@ def _wrap_loop(loop=None):
# The default loop returned by _wrap_loop should be consistent
# with global_event_loop, in order to avoid accidental registration
# of callbacks with a loop that is not intended to run.
- loop = loop or _global_event_loop()
+ loop = loop or _safe_loop()
return (loop if hasattr(loop, '_asyncio_wrapper')
else _AsyncioEventLoop(loop=loop))
@@ -267,8 +264,9 @@ def _safe_loop():
@rtype: asyncio.AbstractEventLoop (or compatible)
@return: event loop instance
"""
- if portage._internal_caller or threading.current_thread() is
threading.main_thread():
- return _global_event_loop()
+ loop = _get_running_loop()
+ if loop is not None:
+ return loop
thread_key = threading.get_ident()
with _thread_weakrefs.lock:
@@ -286,6 +284,16 @@ def _safe_loop():
return loop
+def _get_running_loop():
+ with _thread_weakrefs.lock:
+ if _thread_weakrefs.pid == portage.getpid():
+ try:
+ loop =
_thread_weakrefs.loops[threading.get_ident()]
+ except KeyError:
+ return None
+ return loop if loop.is_running() else None
+
+
def _thread_weakrefs_atexit():
with _thread_weakrefs.lock:
if _thread_weakrefs.pid == portage.getpid():
--
2.26.2
Re: [gentoo-portage-dev] [PATCH gentoolkit] eclean: Add --changed-iuse flag
On 1/2/21 4:08 PM, Matt Turner wrote: > Allows binpkgs to be deleted if they are not usable due to IUSE changes. > --- > Just kind of spitballing. I'm not sure about what USE flags we should > ignore or whether it should be configurable, etc. On one hand, deleting > binpkgs that don't have a newly added PYTHON_TARGET option might make > sense if your binhost is configured to rebuild the package. On the > other, you probably don't want to throw out amd64 binpkgs because > abi_riscv_* was added. The special case for abi_* flags is ugly. Why not do emerge emerge --changed-use, and ignore changed IUSE for flags that aren't enabled? -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH gentoolkit] bin: Add merge-driver-ekeyword
On 12/31/20 11:47 AM, Matt Turner wrote: > Since the KEYWORDS=... assignment is a single line, git struggles to > handle conflicts. When rebasing a series of commits that modify the > KEYWORDS=... it's usually easier to throw them away and reapply on the > new tree than it is to manually handle conflicts during the rebase. > > git allows a 'merge driver' program to handle conflicts; this program > handles conflicts in the KEYWORDS=... assignment. E.g., given an ebuild > with these keywords: > > KEYWORDS="~alpha amd64 arm arm64 ~hppa ppc ppc64 x86" > > One developer drops the ~alpha keyword and pushes to gentoo.git, and > another developer stabilizes hppa. Without this merge driver, git > requires the second developer to manually resolve the conflict which is > tedious and prone to mistakes when rebasing a long series of patches. > With the custom merge driver, it automatically resolves the conflict. > > To use the merge driver, configure your gentoo.git as such: > > gentoo.git/.git/config: > > [merge "keywords"] > name = KEYWORDS merge driver > driver = merge-driver-ekeyword %O %A %B %P > > gentoo.git/.git/info/attributes: > > *.ebuild merge=keywords > > Signed-off-by: Matt Turner > --- > v3: Address Zac's feedback: use tempfile.TemporaryDirectory Looks great! -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH gentoolkit] bin: Add merge-driver-ekeyword
On 12/28/20 5:09 PM, Zac Medico wrote:
> On 12/28/20 3:15 PM, Matt Turner wrote:
>> +def apply_keyword_changes(ebuild: str, pathname: str,
>> + changes: List[Tuple[Optional[str],
>> + Optional[str]]]) -> int:
>> +result: int = 0
>> +
>> +# ekeyword will only modify files named *.ebuild, so make a symlink
>> +ebuild_symlink: str = os.path.basename(pathname)
>> +os.symlink(ebuild, ebuild_symlink)
>
> Are we sure that the current working directory is an entirely safe place
> to create this symlink? A simple fix would be to use
> tempfile.TemporaryDirectory to create a temporary directory to hold the
> symlink. Or, we could change ekeyword to assume that an argument is an
> ebuild if os.path.isfile(arg) succeeds.
>
>> +for removals, additions in changes:
>> +args = []
>> +for rem in removals:
>> +# Drop leading '~' and '-' characters and prepend '^'
>> +i = 1 if rem[0] in ('~', '-') else 0
>> +args.append('^' + rem[i:])
>> +if additions:
>> +args.extend(additions)
>> +args.append(ebuild_symlink)
>> +
>> +result = ekeyword.main(args)
Another option is to bypass the ekeyword.main function, like this:
try:
ekeyword.process_ebuild(pathname, list(map(ekeyword.arg_to_op, args))
except Exception:
result = 1
traceback.print_exc()
else:
result = 0
>> +if result != 0:
>> +break
>> +
>> +os.remove(ebuild_symlink)
>> +return result
>
>
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v2] ekeyword: remove .ebuild file suffix requirement (bug 762331)
We'd like to use ekeyword in a git merge driver implementation, but the
files that the driver will pass to ekeyword do not necessarily have a
.ebuild suffix. Therefore, it would be handy to be able to distinguish
ebuild arguments some other way. If the ignorable_arg(arg) function
returns True and os.path.isfile(arg) returns True, then simply assume
that the argument is an ebuild.
Bug: https://bugs.gentoo.org/762331
Signed-off-by: Zac Medico
---
[PATCH v2] fix to respect the ignorable_arg function
pym/gentoolkit/ekeyword/ekeyword.py | 9 ++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/pym/gentoolkit/ekeyword/ekeyword.py
b/pym/gentoolkit/ekeyword/ekeyword.py
index 4e57c09..eeceed4 100755
--- a/pym/gentoolkit/ekeyword/ekeyword.py
+++ b/pym/gentoolkit/ekeyword/ekeyword.py
@@ -244,7 +244,7 @@ def process_content(ebuild, data, ops, arch_status=None,
verbose=0,
pass
else:
# Chop the full path and the .ebuild suffix.
- disp_name = os.path.basename(ebuild)[:-7]
+ disp_name, _, _ = os.path.basename(ebuild).partition('.ebuild')
def logit(msg):
print('%s: %s' % (disp_name, msg))
@@ -395,7 +395,9 @@ def args_to_work(args, arch_status=None, _repo=None,
quiet=0):
last_todo_arches = []
for arg in args:
- if arg.endswith('.ebuild'):
+ if ignorable_arg(arg, quiet=quiet):
+ pass
+ elif os.path.isfile(arg):
if not todo_arches:
todo_arches = last_todo_arches
work.append([arg, todo_arches])
@@ -405,7 +407,7 @@ def args_to_work(args, arch_status=None, _repo=None,
quiet=0):
op = arg_to_op(arg)
if not arch_status or op.arch in arch_status:
todo_arches.append(op)
- elif not ignorable_arg(arg, quiet=quiet):
+ else:
raise ValueError('unknown arch/argument: %s' %
arg)
if todo_arches:
@@ -475,6 +477,7 @@ def main(argv):
opts.style = 'color-inline'
arch_status = load_profile_data()
+ print(arch_status)
try:
work = args_to_work(work_args, arch_status=arch_status,
quiet=opts.quiet)
except ValueError as e:
--
2.26.2
[gentoo-portage-dev] [PATCH] ekeyword: remove .ebuild file suffix requirement (bug 762331)
We'd like to use ekeyword in a git merge driver implementation, but the
files that the driver will pass to ekeyword do not necessarily have a
.ebuild suffix. Therefore, it would be handy to be able to distinguish
ebuild arguments some other way.
Bug: https://bugs.gentoo.org/762331
Signed-off-by: Zac Medico
---
pym/gentoolkit/ekeyword/ekeyword.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/gentoolkit/ekeyword/ekeyword.py
b/pym/gentoolkit/ekeyword/ekeyword.py
index 4e57c09..665eee5 100755
--- a/pym/gentoolkit/ekeyword/ekeyword.py
+++ b/pym/gentoolkit/ekeyword/ekeyword.py
@@ -244,7 +244,7 @@ def process_content(ebuild, data, ops, arch_status=None,
verbose=0,
pass
else:
# Chop the full path and the .ebuild suffix.
- disp_name = os.path.basename(ebuild)[:-7]
+ disp_name, _, _ = os.path.basename(ebuild).partition('.ebuild')
def logit(msg):
print('%s: %s' % (disp_name, msg))
@@ -395,7 +395,7 @@ def args_to_work(args, arch_status=None, _repo=None,
quiet=0):
last_todo_arches = []
for arg in args:
- if arg.endswith('.ebuild'):
+ if os.path.isfile(arg):
if not todo_arches:
todo_arches = last_todo_arches
work.append([arg, todo_arches])
--
2.26.2
Re: [gentoo-portage-dev] [PATCH gentoolkit] bin: Add merge-driver-ekeyword
On 12/28/20 3:15 PM, Matt Turner wrote:
> +def apply_keyword_changes(ebuild: str, pathname: str,
> + changes: List[Tuple[Optional[str],
> + Optional[str]]]) -> int:
> +result: int = 0
> +
> +# ekeyword will only modify files named *.ebuild, so make a symlink
> +ebuild_symlink: str = os.path.basename(pathname)
> +os.symlink(ebuild, ebuild_symlink)
Are we sure that the current working directory is an entirely safe place
to create this symlink? A simple fix would be to use
tempfile.TemporaryDirectory to create a temporary directory to hold the
symlink. Or, we could change ekeyword to assume that an argument is an
ebuild if os.path.isfile(arg) succeeds.
> +for removals, additions in changes:
> +args = []
> +for rem in removals:
> +# Drop leading '~' and '-' characters and prepend '^'
> +i = 1 if rem[0] in ('~', '-') else 0
> +args.append('^' + rem[i:])
> +if additions:
> +args.extend(additions)
> +args.append(ebuild_symlink)
> +
> +result = ekeyword.main(args)
> +if result != 0:
> +break
> +
> +os.remove(ebuild_symlink)
> +return result
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Drop Python 2 comatibility in extension modules
On 12/24/20 10:14 AM, Mike Gilbert wrote: > Signed-off-by: Mike Gilbert > --- > src/portage_util_file_copy_reflink_linux.c | 10 +- > src/portage_util_libc.c| 10 +- > 2 files changed, 2 insertions(+), 18 deletions(-) Looks good. Please merge. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Adjust mangling of "arch" value from scanelf output
On 12/23/20 7:34 PM, Mike Gilbert wrote:
> scanelf may generate output that looks like this:
>
> ```
> UNKNOWN_TYPE;lib/firmware/ath10k/WCN3990/hw1.0/wlanmdsp.mbn;; - ;
> EM_ARM;lib/firmware/mediatek/mt8183/scp.img;; - ;
> ...
> ```
>
> Previously, we removed the first 3 characters of the first field and
> stored this as the "arch" in NEEDED.ELF.2. This unintentionally
> changes "UNKNOWN_TYPE" to "NOWN_TYPE".
>
> Instead, let's just remove the string "EM_" from the front.
>
> Signed-off-by: Mike Gilbert
> ---
> bin/misc-functions.sh | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
> index c2a16cbe0..d7009d7eb 100755
> --- a/bin/misc-functions.sh
> +++ b/bin/misc-functions.sh
> @@ -194,7 +194,7 @@ install_qa_check() {
> fi
>
> echo "${obj} ${needed}" >>
> "${PORTAGE_BUILDDIR}"/build-info/NEEDED
> - echo "${arch:3};${obj};${soname};${rpath};${needed}" >>
> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2
> + echo "${arch#EM_};${obj};${soname};${rpath};${needed}"
> >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2
> done }
>
> [ -n "${QA_SONAME_NO_SYMLINK}" ] && \
>
Look good. This won't cause any problems for portage since these files
are outside of the known multilib categories.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH gentoolkit 1/4] Remove imports from __future__
On 12/20/20 2:10 PM, Matt Turner wrote: > gentoolkit supports only Python 3.6+ now, so these are not used. > > Signed-off-by: Matt Turner The whole series LGTM (including "Remove unused sys imports" patch 5/4). -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v4] Use default asyncio event loop implementation in API consumer threads
Make the _safe_loop function return an AsyncioEventLoop instance, so that the default asyncio event loop implementation will be used in API consumer threads. This is possible because the underlying asyncio.get_event_loop() function returns a separate event loop for each thread. The AsyncioEventLoop _run_until_complete method will now appropriately handle a ValueError from signal.set_wakeup_fd(-1) if it is not called in the main thread. For external API consumers calling from a non-main thread, an asyncio loop must be registered for the current thread, or else an error will be raised like this: RuntimeError: There is no current event loop in thread 'Thread-1'. In order to avoid this RuntimeError, the external API consumer is responsible for setting an event loop and managing its lifecycle. This code will set an event loop for the current thread: asyncio.set_event_loop(asyncio.new_event_loop()) In order to avoid a ResourceWarning, the caller should also close the corresponding loop before the current thread terminates. Bug: https://bugs.gentoo.org/758755 Signed-off-by: Zac Medico --- [PATCH v4] treat external API consumers the same as interal callers if they call from the main thread, and document asyncio loop lifecycle management now required for external API consumers calling from a non-main thread .../util/_eventloop/asyncio_event_loop.py | 6 - lib/portage/util/futures/_asyncio/__init__.py | 26 ++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index 836f1c30a..4d7047ae8 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -121,4 +121,8 @@ class AsyncioEventLoop(_AbstractEventLoop): try: return self._loop.run_until_complete(future) finally: - self._wakeup_fd = signal.set_wakeup_fd(-1) + try: + self._wakeup_fd = signal.set_wakeup_fd(-1) + except ValueError: + # This is intended to fail when not called in the main thread. + pass diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index a902ad895..0b35c6daf 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -34,7 +34,6 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.futures.unix_events:_PortageEventLoopPolicy', 'portage.util.futures:compat_coroutine@_compat_coroutine', - 'portage.util._eventloop.EventLoop:EventLoop@_EventLoop', ) from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as _AsyncioEventLoop from portage.util._eventloop.global_event_loop import ( @@ -246,14 +245,27 @@ def _wrap_loop(loop=None): def _safe_loop(): """ Return an event loop that's safe to use within the current context. - For portage internal callers, this returns a globally shared event - loop instance. For external API consumers, this constructs a - temporary event loop instance that's safe to use in a non-main - thread (it does not override the global SIGCHLD handler). + For portage internal callers or external API consumers calling from + the main thread, this returns a globally shared event loop instance. + + For external API consumers calling from a non-main thread, an + asyncio loop must be registered for the current thread, or else an + error will be raised like this: + + RuntimeError: There is no current event loop in thread 'Thread-1'. + + In order to avoid this RuntimeError, the external API consumer + is responsible for setting an event loop and managing its lifecycle. + This code will set an event loop for the current thread: + + asyncio.set_event_loop(asyncio.new_event_loop()) + + In order to avoid a ResourceWarning, the caller should also close the + corresponding loop before the current thread terminates. @rtype: asyncio.AbstractEventLoop (or compatible) @return: event loop instance """ - if portage._internal_caller: + if portage._internal_caller or threading.current_thread() is threading.main_thread(): return _global_event_loop() - return _EventLoop(main=False) + return _AsyncioEventLoop() -- 2.26.2
[gentoo-portage-dev] Re: [PATCH v3] Use default asyncio event loop implementation in API consumer threads
Accidentally encrypted the last email. Here's an unencrypted version.
On 12/6/20 2:14 PM, Zac Medico wrote:
> Make the _safe_loop function return an AsyncioEventLoop instance,
> so that the default asyncio event loop implementation will be used
> in API consumer threads. This is possible because the underlying
> asyncio.get_event_loop() function returns a new event loop for
> each thread. The AsyncioEventLoop _run_until_complete method will
> now appropriately handle a ValueError from signal.set_wakeup_fd(-1)
> if it is not called in the main thread.
>
> Bug: https://bugs.gentoo.org/758755
> Signed-off-by: Zac Medico
> ---
> [PATCH v3] fixed AsyncioEventLoop _run_until_complete method to
> handle ValueError from signal.set_wakeup_fd(-1)
>
> lib/portage/util/_eventloop/asyncio_event_loop.py | 6 +-
> lib/portage/util/futures/_asyncio/__init__.py | 3 +--
> 2 files changed, 6 insertions(+), 3 deletions(-)
>
> diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py
> b/lib/portage/util/_eventloop/asyncio_event_loop.py
> index 836f1c30a..4d7047ae8 100644
> --- a/lib/portage/util/_eventloop/asyncio_event_loop.py
> +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
> @@ -121,4 +121,8 @@ class AsyncioEventLoop(_AbstractEventLoop):
> try:
> return self._loop.run_until_complete(future)
> finally:
> - self._wakeup_fd = signal.set_wakeup_fd(-1)
> + try:
> + self._wakeup_fd = signal.set_wakeup_fd(-1)
> + except ValueError:
> + # This is intended to fail when not called in
> the main thread.
> + pass
> diff --git a/lib/portage/util/futures/_asyncio/__init__.py
> b/lib/portage/util/futures/_asyncio/__init__.py
> index a902ad895..12013be00 100644
> --- a/lib/portage/util/futures/_asyncio/__init__.py
> +++ b/lib/portage/util/futures/_asyncio/__init__.py
> @@ -34,7 +34,6 @@ import portage
> portage.proxy.lazyimport.lazyimport(globals(),
> 'portage.util.futures.unix_events:_PortageEventLoopPolicy',
> 'portage.util.futures:compat_coroutine@_compat_coroutine',
> - 'portage.util._eventloop.EventLoop:EventLoop@_EventLoop',
> )
> from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as
> _AsyncioEventLoop
> from portage.util._eventloop.global_event_loop import (
> @@ -256,4 +255,4 @@ def _safe_loop():
> """
> if portage._internal_caller:
> return _global_event_loop()
> - return _EventLoop(main=False)
> + return _AsyncioEventLoop()
>
This fails if an event loop has not been created for the current thread:
File "/usr/lib/python3.8/asyncio/events.py", line 639, in get_event_loop
raise RuntimeError('There is no current event loop in thread %r.'
RuntimeError: There is no current event loop in thread 'Thread-1'.
However, if we automatically instantiate a loop for the current thread
then we will be responsible for closing it as well, or else we'll
eventually see a ResourceWarning like this:
/usr/lib/python3.8/asyncio/base_events.py:654: ResourceWarning: unclosed
event loop <_UnixSelectorEventLoop running=False closed=False debug=False>
_warn(f"unclosed event loop {self!r}", ResourceWarning, source=self)
ResourceWarning: Enable tracemalloc to get the object allocation traceback
So, I think it's probably best if we force the API consumer to manage
the lifecycle of an asyncio loop for each thread that it uses to call
the portage API.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-portage-dev] Re: [PATCH v3] Use default asyncio event loop implementation in API consumer threads
On 12/6/20 2:14 PM, Zac Medico wrote:
> Make the _safe_loop function return an AsyncioEventLoop instance,
> so that the default asyncio event loop implementation will be used
> in API consumer threads. This is possible because the underlying
> asyncio.get_event_loop() function returns a new event loop for
> each thread. The AsyncioEventLoop _run_until_complete method will
> now appropriately handle a ValueError from signal.set_wakeup_fd(-1)
> if it is not called in the main thread.
>
> Bug: https://bugs.gentoo.org/758755
> Signed-off-by: Zac Medico
> ---
> [PATCH v3] fixed AsyncioEventLoop _run_until_complete method to
> handle ValueError from signal.set_wakeup_fd(-1)
>
> lib/portage/util/_eventloop/asyncio_event_loop.py | 6 +-
> lib/portage/util/futures/_asyncio/__init__.py | 3 +--
> 2 files changed, 6 insertions(+), 3 deletions(-)
>
> diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py
> b/lib/portage/util/_eventloop/asyncio_event_loop.py
> index 836f1c30a..4d7047ae8 100644
> --- a/lib/portage/util/_eventloop/asyncio_event_loop.py
> +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
> @@ -121,4 +121,8 @@ class AsyncioEventLoop(_AbstractEventLoop):
> try:
> return self._loop.run_until_complete(future)
> finally:
> - self._wakeup_fd = signal.set_wakeup_fd(-1)
> + try:
> + self._wakeup_fd = signal.set_wakeup_fd(-1)
> + except ValueError:
> + # This is intended to fail when not called in
> the main thread.
> + pass
> diff --git a/lib/portage/util/futures/_asyncio/__init__.py
> b/lib/portage/util/futures/_asyncio/__init__.py
> index a902ad895..12013be00 100644
> --- a/lib/portage/util/futures/_asyncio/__init__.py
> +++ b/lib/portage/util/futures/_asyncio/__init__.py
> @@ -34,7 +34,6 @@ import portage
> portage.proxy.lazyimport.lazyimport(globals(),
> 'portage.util.futures.unix_events:_PortageEventLoopPolicy',
> 'portage.util.futures:compat_coroutine@_compat_coroutine',
> - 'portage.util._eventloop.EventLoop:EventLoop@_EventLoop',
> )
> from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as
> _AsyncioEventLoop
> from portage.util._eventloop.global_event_loop import (
> @@ -256,4 +255,4 @@ def _safe_loop():
> """
> if portage._internal_caller:
> return _global_event_loop()
> - return _EventLoop(main=False)
> + return _AsyncioEventLoop()
>
This fails if an event loop has not been created for the current thread:
File "/usr/lib/python3.8/asyncio/events.py", line 639, in get_event_loop
raise RuntimeError('There is no current event loop in thread %r.'
RuntimeError: There is no current event loop in thread 'Thread-1'.
However, if we automatically instantiate a loop for the current thread
then we will be responsible for closing it as well, or else we'll
eventually see a ResourceWarning like this:
/usr/lib/python3.8/asyncio/base_events.py:654: ResourceWarning: unclosed
event loop <_UnixSelectorEventLoop running=False closed=False debug=False>
_warn(f"unclosed event loop {self!r}", ResourceWarning, source=self)
ResourceWarning: Enable tracemalloc to get the object allocation traceback
So, I think it's probably best if we force the API consumer to manage
the lifecycle of an asyncio loop for each thread that it uses to call
the portage API.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v3] Use default asyncio event loop implementation in API consumer threads
Make the _safe_loop function return an AsyncioEventLoop instance, so that the default asyncio event loop implementation will be used in API consumer threads. This is possible because the underlying asyncio.get_event_loop() function returns a new event loop for each thread. The AsyncioEventLoop _run_until_complete method will now appropriately handle a ValueError from signal.set_wakeup_fd(-1) if it is not called in the main thread. Bug: https://bugs.gentoo.org/758755 Signed-off-by: Zac Medico --- [PATCH v3] fixed AsyncioEventLoop _run_until_complete method to handle ValueError from signal.set_wakeup_fd(-1) lib/portage/util/_eventloop/asyncio_event_loop.py | 6 +- lib/portage/util/futures/_asyncio/__init__.py | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index 836f1c30a..4d7047ae8 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -121,4 +121,8 @@ class AsyncioEventLoop(_AbstractEventLoop): try: return self._loop.run_until_complete(future) finally: - self._wakeup_fd = signal.set_wakeup_fd(-1) + try: + self._wakeup_fd = signal.set_wakeup_fd(-1) + except ValueError: + # This is intended to fail when not called in the main thread. + pass diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index a902ad895..12013be00 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -34,7 +34,6 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.futures.unix_events:_PortageEventLoopPolicy', 'portage.util.futures:compat_coroutine@_compat_coroutine', - 'portage.util._eventloop.EventLoop:EventLoop@_EventLoop', ) from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as _AsyncioEventLoop from portage.util._eventloop.global_event_loop import ( @@ -256,4 +255,4 @@ def _safe_loop(): """ if portage._internal_caller: return _global_event_loop() - return _EventLoop(main=False) + return _AsyncioEventLoop() -- 2.26.2
