[gentoo-portage-dev] [PATCH 1/2] Add get_repo_revision_history function and repo_revisions file
The history of synced revisions is provided by a new
get_repo_revision_history function and corresponding
/var/lib/portage/repo_revisions file, with history
limit currently capped at 25 revisions. If a change
is detected and the current process has permission
to update the repo_revisions file, then the file will
be updated with any newly detected revisions.
For volatile repos the revisions may be unordered,
which makes them unusable for the purposes of the
revision history, so the revisions of volatile repos
are not tracked. This functions detects revisions
which are not yet visible to the current process due
to the sync-rcu option.
The emaint revisions --purgerepos and --purgeallrepos
options allow revisions for some or all repos to be
easily purged from the history. For example, the
emerge-webrsync script uses this emaint commmand to
purge the revision history of the gentoo repo when
the emerge-webrsync --revert option is used to roll
back to a previous snapshot:
emaint revisions --purgerepos="${repo_name}"
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico
---
bin/emerge-webrsync | 3 +-
lib/portage/const.py | 1 +
lib/portage/emaint/modules/meson.build| 1 +
.../emaint/modules/revisions/__init__.py | 36 +
.../emaint/modules/revisions/meson.build | 8 ++
.../emaint/modules/revisions/revisions.py | 95 +
lib/portage/sync/controller.py| 8 +-
lib/portage/sync/meson.build | 1 +
lib/portage/sync/revision_history.py | 133 ++
lib/portage/tests/sync/test_sync_local.py | 75 +-
man/emaint.1 | 18 ++-
man/portage.5 | 17 ++-
12 files changed, 388 insertions(+), 8 deletions(-)
create mode 100644 lib/portage/emaint/modules/revisions/__init__.py
create mode 100644 lib/portage/emaint/modules/revisions/meson.build
create mode 100644 lib/portage/emaint/modules/revisions/revisions.py
create mode 100644 lib/portage/sync/revision_history.py
diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync
index 99da05543a..caa4986da2 100755
--- a/bin/emerge-webrsync
+++ b/bin/emerge-webrsync
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# Author: Karl Trygve Kalleberg
# Rewritten from the old, Perl-based emerge-webrsync script
@@ -732,6 +732,7 @@ main() {
[[ ${do_debug} -eq 1 ]] && set -x
if [[ -n ${revert_date} ]] ; then
+ emaint revisions --purgerepos="${repo_name}"
do_snapshot 1 "${revert_date}"
else
do_latest_snapshot
diff --git a/lib/portage/const.py b/lib/portage/const.py
index 2154213b7b..c9a71009a7 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -51,6 +51,7 @@ PRIVATE_PATH = "var/lib/portage"
WORLD_FILE = f"{PRIVATE_PATH}/world"
WORLD_SETS_FILE = f"{PRIVATE_PATH}/world_sets"
CONFIG_MEMORY_FILE = f"{PRIVATE_PATH}/config"
+REPO_REVISIONS = f"{PRIVATE_PATH}/repo_revisions"
NEWS_LIB_PATH = "var/lib/gentoo"
# these variables get EPREFIX prepended automagically when they are
diff --git a/lib/portage/emaint/modules/meson.build
b/lib/portage/emaint/modules/meson.build
index 48f4f77d83..33b396be94 100644
--- a/lib/portage/emaint/modules/meson.build
+++ b/lib/portage/emaint/modules/meson.build
@@ -12,5 +12,6 @@ subdir('logs')
subdir('merges')
subdir('move')
subdir('resume')
+subdir('revisions')
subdir('sync')
subdir('world')
diff --git a/lib/portage/emaint/modules/revisions/__init__.py
b/lib/portage/emaint/modules/revisions/__init__.py
new file mode 100644
index 00..c51cbb4bf3
--- /dev/null
+++ b/lib/portage/emaint/modules/revisions/__init__.py
@@ -0,0 +1,36 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+doc = """Purge repo_revisions history file."""
+__doc__ = doc
+
+
+module_spec = {
+"name": "revisions",
+"description": doc,
+"provides": {
+"purgerevisions": {
+"name": "revisions",
+"sourcefile": "revisions",
+"class": "PurgeRevisions",
+"description": "Purge repo_revisions history",
+"functions": ["purgeallrepos", "purgerepos"],
+"func_desc": {
+"repo": {
+"long": "--purgerepos",
+"help": "(revisions module only): --purgerepos Purge
revisions for the specif
[gentoo-portage-dev] [PATCH 2/2] bintree: Add REPO_REVISIONS to package index header
As a means for binhost clients to select source repo
revisions which are consistent with binhosts, inject
REPO_REVISIONS from a package into the index header,
using a history of synced revisions to guarantee
forward progress. This queries the relevant repos to
check if any new revisions have appeared in the
absence of a proper sync operation.
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico
---
lib/portage/dbapi/bintree.py | 66 -
lib/portage/tests/sync/test_sync_local.py | 71 +++
2 files changed, 123 insertions(+), 14 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 7bc1f60f6d..fbf60e74eb 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -48,6 +48,7 @@ from portage.exception import (
from portage.localization import _
from portage.output import colorize
from portage.package.ebuild.profile_iuse import iter_iuse_vars
+from portage.sync.revision_history import get_repo_revision_history
from portage.util import ensure_dirs
from portage.util.file_copy import copyfile
from portage.util.futures import asyncio
@@ -62,6 +63,7 @@ from portage import _unicode_encode
import codecs
import errno
import io
+import json
import re
import stat
import subprocess
@@ -134,13 +136,19 @@ class bindbapi(fakedbapi):
"USE",
"_mtime_",
}
+# Keys required only when initially adding a package.
+self._init_aux_keys = {
+"REPO_REVISIONS",
+}
self._aux_cache = {}
self._aux_cache_slot_dict_cache = None
@property
def _aux_cache_slot_dict(self):
if self._aux_cache_slot_dict_cache is None:
-self._aux_cache_slot_dict_cache =
slot_dict_class(self._aux_cache_keys)
+self._aux_cache_slot_dict_cache = slot_dict_class(
+chain(self._aux_cache_keys, self._init_aux_keys)
+)
return self._aux_cache_slot_dict_cache
def __getstate__(self):
@@ -1791,6 +1799,10 @@ class binarytree:
pkgindex = self._new_pkgindex()
d = self._inject_file(pkgindex, cpv, full_path)
+repo_revisions = d.get("REPO_REVISIONS")
+if repo_revisions:
+repo_revisions = json.loads(repo_revisions)
+self._inject_repo_revisions(pkgindex.header, repo_revisions)
self._update_pkgindex_header(pkgindex.header)
self._pkgindex_write(pkgindex)
@@ -1872,7 +1884,7 @@ class binarytree:
@return: package metadata
"""
if keys is None:
-keys = self.dbapi._aux_cache_keys
+keys = chain(self.dbapi._aux_cache_keys, self.dbapi._init_aux_keys)
metadata = self.dbapi._aux_cache_slot_dict()
else:
metadata = {}
@@ -1916,6 +1928,56 @@ class binarytree:
return metadata
+def _inject_repo_revisions(self, header, repo_revisions):
+"""
+Inject REPO_REVISIONS from a package into the index header,
+using a history of synced revisions to guarantee forward
+progress. This queries the relevant repos to check if any
+new revisions have appeared in the absence of a proper sync
+operation.
+
+This does not expose REPO_REVISIONS that do not appear in
+the sync history, since such revisions suggest that the
+package was not built locally, and in this case its
+REPO_REVISIONS are not intended to be exposed.
+"""
+synced_repo_revisions = get_repo_revision_history(
+self.settings["EROOT"],
+[self.settings.repositories[repo_name] for repo_name in
repo_revisions],
+)
+header_repo_revisions = (
+json.loads(header["REPO_REVISIONS"]) if
header.get("REPO_REVISIONS") else {}
+)
+for repo_name, repo_revision in repo_revisions.items():
+rev_list = synced_repo_revisions.get(repo_name, [])
+header_rev = header_repo_revisions.get(repo_name)
+if not rev_list or header_rev in (repo_revision, rev_list[0]):
+continue
+try:
+header_rev_index = (
+None if header_rev is None else rev_list.index(header_rev)
+)
+except ValueError:
+header_rev_index = None
+try:
+repo_revision_index = rev_list.index(repo_revision)
+except ValueError:
+repo_revision_index = None
+if repo_revision_index is not None and (
+header_rev_index is None or repo_revision_index <
header_rev_index
+):
+# There is forward progress when repo_revision is more recent
+# than header_rev
[gentoo-portage-dev] [PATCH 0/2] Add REPO_REVISIONS to package index header
As a means for binhost clients to select source repo
revisions which are consistent with binhosts, inject
REPO_REVISIONS from a package into the index header,
using a history of synced revisions to guarantee
forward progress. This queries the relevant repos to
check if any new revisions have appeared in the
absence of a proper sync operation.
The history of synced revisions is provided by a new
get_repo_revision_history function and corresponding
/var/lib/portage/repo_revisions file, with history
limit currently capped at 25 revisions. If a change
is detected and the current process has permission
to update the repo_revisions file, then the file will
be updated with any newly detected revisions.
For volatile repos the revisions may be unordered,
which makes them unusable for the purposes of the
revision history, so the revisions of volatile repos
are not tracked. This functions detects revisions
which are not yet visible to the current process due
to the sync-rcu option.
The emaint revisions --purgerepos and --purgeallrepos
options allow revisions for some or all repos to be
easily purged from the history. For example, the
emerge-webrsync script uses this emaint commmand to
purge the revision history of the gentoo repo when
the emerge-webrsync --revert option is used to roll
back to a previous snapshot:
emaint revisions --purgerepos="${repo_name}"
This series can also be reviewed at https://github.com/gentoo/portage/pull/1306.
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico
Zac Medico (2):
Add get_repo_revision_history function and repo_revisions file
bintree: Add REPO_REVISIONS to package index header
bin/emerge-webrsync | 3 +-
lib/portage/const.py | 1 +
lib/portage/dbapi/bintree.py | 66 +++-
lib/portage/emaint/modules/meson.build| 1 +
.../emaint/modules/revisions/__init__.py | 36 +
.../emaint/modules/revisions/meson.build | 8 +
.../emaint/modules/revisions/revisions.py | 95
lib/portage/sync/controller.py| 8 +-
lib/portage/sync/meson.build | 1 +
lib/portage/sync/revision_history.py | 133
lib/portage/tests/sync/test_sync_local.py | 146 --
man/emaint.1 | 18 ++-
man/portage.5 | 17 +-
13 files changed, 511 insertions(+), 22 deletions(-)
create mode 100644 lib/portage/emaint/modules/revisions/__init__.py
create mode 100644 lib/portage/emaint/modules/revisions/meson.build
create mode 100644 lib/portage/emaint/modules/revisions/revisions.py
create mode 100644 lib/portage/sync/revision_history.py
--
2.41.0
[gentoo-portage-dev] [PATCH 9/9] EbuildPhase: async_check_locale
Change config.environ() check_locale calls to async_check_locale
calls in the EbuildPhase _async_start method in order to eliminate
synchronous waiting for child processes in the main event loop
thread.
Bug: https://bugs.gentoo.org/923841
Signed-off-by: Zac Medico
---
lib/_emerge/EbuildMetadataPhase.py| 4 +++
lib/_emerge/EbuildPhase.py| 28 ++-
lib/portage/package/ebuild/config.py | 26 -
lib/portage/util/futures/_asyncio/__init__.py | 9 ++
lib/portage/util/locale.py| 28 +--
5 files changed, 70 insertions(+), 25 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index 9fcdabe840..90a3ea05aa 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,6 +8,7 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"_emerge.EbuildPhase:_setup_locale",
"portage.package.ebuild._metadata_invalid:eapi_invalid",
)
from portage import os
@@ -83,6 +84,9 @@ class EbuildMetadataPhase(SubProcess):
settings.setcpv(self.cpv)
settings.configdict["pkg"]["EAPI"] = parsed_eapi
+# This requires above setcpv and EAPI setup.
+await _setup_locale(self.settings)
+
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index c81bf54a81..c8caf73722 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import (
_prepare_fake_distdir,
_prepare_fake_filesdir,
)
+from portage.eapi import _get_eapi_attrs
from portage.util import writemsg, ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport(
+ "_post_src_install_write_metadata,"
+ "_preinst_bsdflags",
"portage.util.futures.unix_events:_set_nonblocking",
+"portage.util.locale:async_check_locale,split_LC_ALL",
)
from portage import os
from portage import _encodings
from portage import _unicode_encode
+async def _setup_locale(settings):
+eapi_attrs = _get_eapi_attrs(settings["EAPI"])
+if eapi_attrs.posixish_locale:
+split_LC_ALL(settings)
+settings["LC_COLLATE"] = "C"
+# check_locale() returns None when check can not be executed.
+if await async_check_locale(silent=True, env=settings.environ()) is
False:
+# try another locale
+for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
+settings["LC_CTYPE"] = l
+if await async_check_locale(silent=True,
env=settings.environ()):
+# TODO: output the following only once
+# writemsg(
+# _("!!! LC_CTYPE unsupported, using %s instead\n")
+# % self.settings["LC_CTYPE"]
+# )
+break
+else:
+raise AssertionError("C locale did not pass the test!")
+
+
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") +
("_ebuild_lock",)
@@ -94,6 +117,9 @@ class EbuildPhase(CompositeTask):
self._start_task(AsyncTaskFuture(future=future),
self._async_start_exit)
async def _async_start(self):
+
+await _setup_locale(self.settings)
+
need_builddir = self.phase not in
EbuildProcess._phases_without_builddir
if need_builddir:
diff --git a/lib/portage/package/ebuild/config.py
b/lib/portage/package/ebuild/config.py
index d7b0ca5676..35c77486ec 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -29,7 +29,6 @@ portage.proxy.lazyimport.lazyimport(
"portage.dbapi.vartree:vartree",
"portage.package.ebuild.doebuild:_phase_func_map",
"portage.util.compression_probe:_compressors",
-"portage.util.locale:check_locale,split_LC_ALL",
)
from portage import bsd_chflags, load_mod, os, selinux, _unicode_decode
from portage.const import (
@@ -3368,20 +3367,17 @@ class config:
mydict["EBUILD_PHASE_FUNC"] = phase_func
if eapi_attrs.posixish_locale:
-split_LC_ALL(mydict)
-mydict["LC_COLLATE"] = "C"
-
[gentoo-portage-dev] [PATCH 8/9] EbuildMetadataPhase: Migrate to _async_start
Bug: https://bugs.gentoo.org/923841 Signed-off-by: Zac Medico --- lib/_emerge/EbuildMetadataPhase.py | 17 ++--- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/_emerge/EbuildMetadataPhase.py b/lib/_emerge/EbuildMetadataPhase.py index 784712e8cb..9fcdabe840 100644 --- a/lib/_emerge/EbuildMetadataPhase.py +++ b/lib/_emerge/EbuildMetadataPhase.py @@ -46,6 +46,12 @@ class EbuildMetadataPhase(SubProcess): _files_dict = slot_dict_class(_file_names, prefix="") def _start(self): +asyncio.ensure_future( +self._async_start(), loop=self.scheduler +).add_done_callback(self._async_start_done) +self._registered = True + +async def _async_start(self): ebuild_path = self.ebuild_hash.location with open( @@ -116,7 +122,6 @@ class EbuildMetadataPhase(SubProcess): self._raw_metadata = [] files.ebuild = master_fd self.scheduler.add_reader(files.ebuild, self._output_handler) -self._registered = True retval = portage.doebuild( ebuild_path, @@ -150,16 +155,6 @@ class EbuildMetadataPhase(SubProcess): self._proc = retval -asyncio.ensure_future( -self._async_start(), loop=self.scheduler -).add_done_callback(self._async_start_done) - -async def _async_start(self): -# Call async check_locale here for bug 923841, but code -# also needs to migrate from _start to here, including -# the self.deallocate_config set_result call. -pass - def _async_start_done(self, future): future.cancelled() or future.result() if self._was_cancelled(): -- 2.41.0
[gentoo-portage-dev] [PATCH 7/9] actions: disable pytest-xdist for spawn start-method (workers crash)
Bug: https://bugs.gentoo.org/924416
Signed-off-by: Zac Medico
---
.github/workflows/ci.yml | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5bffd97206..762999b7cc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -90,5 +90,8 @@ jobs:
- name: Run tests for ${{ matrix.python-version }}
run: |
[[ "${{ matrix.start-method }}" == "spawn" ]] && export
PORTAGE_MULTIPROCESSING_START_METHOD=spawn
- export PYTEST_ADDOPTS="-vv -ra -l -o console_output_style=count -n
$(nproc) --dist=worksteal"
+ # spawn start-method crashes pytest-xdist workers (bug 924416)
+ [[ "${{ matrix.start-method }}" == "spawn" ]] && \
+export PYTEST_ADDOPTS="-vv -ra -l -o console_output_style=count"
|| \
+export PYTEST_ADDOPTS="-vv -ra -l -o console_output_style=count -n
$(nproc) --dist=worksteal"
meson test -C /tmp/build --verbose
--
2.41.0
[gentoo-portage-dev] [PATCH 6/9] async_aux_get: Use EbuildMetadataPhase deallocate_config future
For the portdbapi async_aux_get method, there is not a very
good place to store a config pool, so instead use asyncio.Lock
to manage access to the portdbapi doebuild_settings attribute
when using the main event loop in the main thread. For other
threads, clone a config instance since we do not have a
thread-safe config pool. This cloning is expensive, but since
portage internals do not trigger this case, it suffices for now
(an AssertionError ensures that internals do not trigger it).
For the main event loop running in the main thread, performance
with the asyncio.Lock should not be significantly different to
performance prior to commit c95fc64abf96, since check_locale
results are typically cached and before there was only a single
shared doebuild_settings instance with access serialized via
the EbuildMetadataPhase _start method.
Update async_aux_get callers to use asyncio.ensure_future on
the returned coroutine when needed, since it used to return
a future instead of a coroutine, and sometimes a future is
needed for add_done_callback usage.
In the portdbapi async_fetch_map method, fix a broken reference
to "future" which should have been "aux_get_future", an error
discovered while testing this patch.
Bug: https://bugs.gentoo.org/924319
Fixes: c95fc64abf96 ("EbuildPhase: async_check_locale")
Signed-off-by: Zac Medico
---
lib/portage/_emirrordist/FetchIterator.py | 10 +-
lib/portage/dbapi/porttree.py | 129 +++---
lib/portage/tests/update/test_move_ent.py | 3 +
3 files changed, 97 insertions(+), 45 deletions(-)
diff --git a/lib/portage/_emirrordist/FetchIterator.py
b/lib/portage/_emirrordist/FetchIterator.py
index eaf3e53596..e4fdd092af 100644
--- a/lib/portage/_emirrordist/FetchIterator.py
+++ b/lib/portage/_emirrordist/FetchIterator.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2018 Gentoo Foundation
+# Copyright 2013-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import threading
@@ -14,6 +14,7 @@ from portage.exception import PortageException,
PortageKeyError
from portage.package.ebuild.fetch import DistfileName
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.TaskScheduler import TaskScheduler
+from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from .FetchTask import FetchTask
from _emerge.CompositeTask import CompositeTask
@@ -276,8 +277,11 @@ def _async_fetch_tasks(config, hash_filter, repo_config,
digests_future, cpv, lo
result.set_result(fetch_tasks)
def future_generator():
-yield config.portdb.async_aux_get(
-cpv, ("RESTRICT",), myrepo=repo_config.name, loop=loop
+yield asyncio.ensure_future(
+config.portdb.async_aux_get(
+cpv, ("RESTRICT",), myrepo=repo_config.name, loop=loop
+),
+loop,
)
yield config.portdb.async_fetch_map(cpv, mytree=repo_config.location,
loop=loop)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 61d431f917..4eebe1183f 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2021 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["close_portdbapi_caches", "FetchlistDict", "portagetree",
"portdbapi"]
@@ -41,7 +41,9 @@ from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
+import contextlib
import os as _os
+import threading
import traceback
import warnings
import errno
@@ -239,6 +241,7 @@ class portdbapi(dbapi):
# this purpose because doebuild makes many changes to the config
# instance that is passed in.
self.doebuild_settings = config(clone=self.settings)
+self._doebuild_settings_lock = asyncio.Lock()
self.depcachedir = os.path.realpath(self.settings.depcachedir)
if os.environ.get("SANDBOX_ON") == "1":
@@ -356,6 +359,17 @@ class portdbapi(dbapi):
self._better_cache = None
self._broken_ebuilds = set()
+def __getstate__(self):
+state = self.__dict__.copy()
+# These attributes are not picklable, so they are automatically
+# regenerated after unpickling.
+state["_doebuild_settings_lock"] = None
+return state
+
+def __setstate__(self, state):
+self.__dict__.update(state)
+self._doebuild_settings_lock = asyncio.Lock()
+
def _set_porttrees(self, porttrees):
"""
Consumers, such as emirrordist, may modify the porttrees attribute in
@@ -669,7 +683,7 @@ class portdbapi(dbapi):
self.async_aux_get(mycpv, mylist,
[gentoo-portage-dev] [PATCH 5/9] asyncio: Wrap asyncio.Lock for python 3.9 compat
Wrap asyncio.Lock for compatibility with python 3.9 where the
deprecated loop parameter is required in order to avoid "got
Future attached to a different loop" errors.
The pordbapi async_aux_get method can use asyncio.Lock to
serialize access to its doebuild_settings attribute in order
to prevent issues like bug 924319.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico
---
lib/portage/util/futures/_asyncio/__init__.py | 17 +
1 file changed, 17 insertions(+)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index 8f1b8e8275..b6481c281e 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -9,6 +9,7 @@ __all__ = (
"CancelledError",
"Future",
"InvalidStateError",
+"Lock",
"TimeoutError",
"get_child_watcher",
"get_event_loop",
@@ -22,6 +23,7 @@ __all__ = (
"wait_for",
)
+import sys
import types
import weakref
@@ -35,6 +37,7 @@ from asyncio import (
FIRST_EXCEPTION,
Future,
InvalidStateError,
+Lock as _Lock,
shield,
TimeoutError,
wait_for,
@@ -159,6 +162,20 @@ def iscoroutinefunction(func):
return False
+class Lock(_Lock):
+"""
+Inject loop parameter for python3.9 or less in order to avoid
+"got Future attached to a different loop" errors.
+"""
+
+def __init__(self, **kwargs):
+if sys.version_info >= (3, 10):
+kwargs.pop("loop", None)
+elif "loop" not in kwargs:
+kwargs["loop"] = _safe_loop()._loop
+super().__init__(**kwargs)
+
+
class Task(Future):
"""
Schedule the execution of a coroutine: wrap it in a future. A task
--
2.41.0
[gentoo-portage-dev] [PATCH 4/9] ResolverPlayground: Use egencache to create manifests
Make the ResolverPlayground _create_ebuild_manifests method
call egencache --jobs, which reliably triggers the KeyError
from bug 924319 for multiple tests:
lib/portage/tests/bin/test_doins.py::DoIns::testDoInsFallback Exception in
callback EbuildMetadataPhase._async_start_done()
handle: )>
Traceback (most recent call last):
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "lib/_emerge/EbuildMetadataPhase.py", line 154, in _async_start_done
future.cancelled() or future.result()
^^^
File "lib/_emerge/EbuildMetadataPhase.py", line 130, in _async_start
retval = portage.doebuild(
^
File "lib/portage/package/ebuild/doebuild.py", line 1030, in doebuild
doebuild_environment(
File "lib/portage/package/ebuild/doebuild.py", line 519, in
doebuild_environment
eapi = mysettings.configdict["pkg"]["EAPI"]
File "lib/portage/util/__init__.py", line 1684, in __getitem__
return UserDict.__getitem__(self, item_key)
File "lib/portage/cache/mappings.py", line 175, in __getitem__
return self.data[key]
~^
KeyError: 'EAPI'
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico
---
lib/portage/tests/dbapi/test_portdb_cache.py | 4 +-
.../tests/resolver/ResolverPlayground.py | 38 ++-
2 files changed, 23 insertions(+), 19 deletions(-)
diff --git a/lib/portage/tests/dbapi/test_portdb_cache.py
b/lib/portage/tests/dbapi/test_portdb_cache.py
index c7c6913b49..c24a4f2098 100644
--- a/lib/portage/tests/dbapi/test_portdb_cache.py
+++ b/lib/portage/tests/dbapi/test_portdb_cache.py
@@ -1,6 +1,7 @@
-# Copyright 2012-2023 Gentoo Authors
+# Copyright 2012-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import shutil
import subprocess
import sys
import textwrap
@@ -63,6 +64,7 @@ class PortdbCacheTestCase(TestCase):
python_cmd = (portage_python, "-b", "-Wd", "-c")
test_commands = (
+(lambda: shutil.rmtree(md5_cache_dir) or True,),
(lambda: not os.path.exists(pms_cache_dir),),
(lambda: not os.path.exists(md5_cache_dir),),
python_cmd
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index 2d26012873..75c86b615c 100644
--- a/lib/portage/tests/resolver/ResolverPlayground.py
+++ b/lib/portage/tests/resolver/ResolverPlayground.py
@@ -3,6 +3,7 @@
import bz2
import fnmatch
+import subprocess
import tempfile
import portage
@@ -18,8 +19,6 @@ from portage.const import (
from portage.process import find_binary
from portage.dep import Atom, _repo_separator
from portage.dbapi.bintree import binarytree
-from portage.package.ebuild.config import config
-from portage.package.ebuild.digestgen import digestgen
from portage._sets import load_default_config
from portage._sets.base import InternalPackageSet
from portage.tests import cnf_path
@@ -323,22 +322,25 @@ class ResolverPlayground:
f.write(misc_content)
def _create_ebuild_manifests(self, ebuilds):
-tmpsettings = config(clone=self.settings)
-tmpsettings["PORTAGE_QUIET"] = "1"
-for cpv in ebuilds:
-a = Atom("=" + cpv, allow_repo=True)
-repo = a.repo
-if repo is None:
-repo = "test_repo"
-
-repo_dir = self._get_repo_dir(repo)
-ebuild_dir = os.path.join(repo_dir, a.cp)
-ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] +
".ebuild")
-
-portdb = self.trees[self.eroot]["porttree"].dbapi
-tmpsettings["O"] = ebuild_dir
-if not digestgen(mysettings=tmpsettings, myportdb=portdb):
-raise AssertionError(f"digest creation failed for
{ebuild_path}")
+for repo_name in self._repositories:
+if repo_name == "DEFAULT":
+continue
+egencache_cmd = [
+"egencache",
+f"--repo={repo_name}",
+"--update",
+"--update-manifests",
+"--sign-manifests=n",
+"--strict-manifests=n",
+
f"--repositories-configuration={self.settings['PORTAGE_REPOSITORIES']}",
+f"--jobs={portage.util.cpuinfo.get_cpu_count()}",
+]
+result = subprocess.run(
+egencache_cmd,
+env=self.settings.environ(),
+)
+if result.returncode != os.EX_OK:
+raise AssertionError(f"command failed: {egencache_cmd}")
def _create_binpkgs(self, binpkgs):
# When using BUILD_ID, there can be multiple instances for the
--
2.41.0
[gentoo-portage-dev] [PATCH 3/9] MetadataRegen: Use EbuildMetadataPhase deallocate_config
For EbuildMetadataPhase consumers like MetadataRegen and
depgraph, store a pool of config instances in a config_pool
list, and return instaces to the list when the deallocate_config
future is done.
Bug: https://bugs.gentoo.org/924319
Fixes: c95fc64abf96 ("EbuildPhase: async_check_locale")
Signed-off-by: Zac Medico
---
lib/_emerge/MetadataRegen.py | 16 ++--
lib/_emerge/depgraph.py | 11 +++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/lib/_emerge/MetadataRegen.py b/lib/_emerge/MetadataRegen.py
index d29722b94c..538a94b450 100644
--- a/lib/_emerge/MetadataRegen.py
+++ b/lib/_emerge/MetadataRegen.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
@@ -44,6 +44,7 @@ class MetadataRegen(AsyncScheduler):
valid_pkgs = self._valid_pkgs
cp_set = self._cp_set
consumer = self._consumer
+config_pool = []
portage.writemsg_stdout("Regenerating cache entries...\n")
for cp in self._cp_iter:
@@ -73,12 +74,23 @@ class MetadataRegen(AsyncScheduler):
consumer(cpv, repo_path, metadata, ebuild_hash,
True)
continue
+if config_pool:
+settings = config_pool.pop()
+else:
+settings = portage.config(clone=portdb.settings)
+
+deallocate_config = self.scheduler.create_future()
+deallocate_config.add_done_callback(
+lambda future: config_pool.append(future.result())
+)
+
yield EbuildMetadataPhase(
cpv=cpv,
ebuild_hash=ebuild_hash,
portdb=portdb,
repo_path=repo_path,
-settings=portdb.doebuild_settings,
+settings=settings,
+deallocate_config=deallocate_config,
write_auxdb=self._write_auxdb,
)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 1674fa289e..70b83ee1f4 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -754,6 +754,7 @@ class depgraph:
def _dynamic_deps_preload(self, fake_vartree):
portdb = fake_vartree._portdb
+config_pool = []
for pkg in fake_vartree.dbapi:
self._spinner_update()
self._dynamic_config._package_tracker.add_installed_pkg(pkg)
@@ -768,12 +769,22 @@ class depgraph:
if metadata is not None:
fake_vartree.dynamic_deps_preload(pkg, metadata)
else:
+if config_pool:
+settings = config_pool.pop()
+else:
+settings = portage.config(clone=portdb.settings)
+
+deallocate_config = portdb._event_loop.create_future()
+deallocate_config.add_done_callback(
+lambda future: config_pool.append(future.result())
+)
proc = EbuildMetadataPhase(
cpv=pkg.cpv,
ebuild_hash=ebuild_hash,
portdb=portdb,
repo_path=repo_path,
settings=portdb.doebuild_settings,
+deallocate_config=deallocate_config,
)
proc.addExitListener(self._dynamic_deps_proc_exit(pkg,
fake_vartree))
yield proc
--
2.41.0
[gentoo-portage-dev] [PATCH 2/9] EbuildMetadataPhase: Add deallocate_config future
Use a deallocate_config future to release self.settings when
it is no longer needed. It's necessary to manage concurrency
since commit c95fc64abf96 because mutation of self.settings
is no longer limited to the EbuildMetadataPhase _start method,
where exclusive access was guaranteed within the main thread.
Add support to the isAlive() method to detect when the
EbuildMetadataPhase has started but the pid is not yet
available (due to async_check_locale usage from commit
c95fc64abf96). This can be used to check if an
EbuildMetadataPhase instance has been successfully started
so that it can be relied upon to set the result of the
deallocate_config future.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico
---
lib/_emerge/EbuildMetadataPhase.py | 36 ++
lib/_emerge/SubProcess.py | 5 -
2 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index f4f685e81c..784712e8cb 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -14,6 +14,7 @@ from portage import os
from portage import _encodings
from portage import _unicode_decode
from portage import _unicode_encode
+from portage.util.futures import asyncio
import fcntl
@@ -33,6 +34,7 @@ class EbuildMetadataPhase(SubProcess):
"portdb",
"repo_path",
"settings",
+"deallocate_config",
"write_auxdb",
) + (
"_eapi",
@@ -127,6 +129,15 @@ class EbuildMetadataPhase(SubProcess):
returnproc=True,
)
settings.pop("PORTAGE_PIPE_FD", None)
+# At this point we can return settings to the caller
+# since we never use it for anything more than an
+# eapi_invalid call after this, and eapi_invalid is
+# insensitive to concurrent modifications.
+if (
+self.deallocate_config is not None
+and not self.deallocate_config.cancelled()
+):
+self.deallocate_config.set_result(settings)
os.close(slave_fd)
null_input.close()
@@ -139,6 +150,31 @@ class EbuildMetadataPhase(SubProcess):
self._proc = retval
+asyncio.ensure_future(
+self._async_start(), loop=self.scheduler
+).add_done_callback(self._async_start_done)
+
+async def _async_start(self):
+# Call async check_locale here for bug 923841, but code
+# also needs to migrate from _start to here, including
+# the self.deallocate_config set_result call.
+pass
+
+def _async_start_done(self, future):
+future.cancelled() or future.result()
+if self._was_cancelled():
+pass
+elif future.cancelled():
+self.cancel()
+self._was_cancelled()
+
+if self.deallocate_config is not None and not
self.deallocate_config.done():
+self.deallocate_config.set_result(self.settings)
+
+if self.returncode is not None:
+self._unregister()
+self.wait()
+
def _output_handler(self):
while True:
buf = self._read_buf(self._files.ebuild)
diff --git a/lib/_emerge/SubProcess.py b/lib/_emerge/SubProcess.py
index 029bbc3f44..057e0adc24 100644
--- a/lib/_emerge/SubProcess.py
+++ b/lib/_emerge/SubProcess.py
@@ -18,9 +18,12 @@ class SubProcess(AbstractPollTask):
# we've sent a kill signal to our subprocess.
_cancel_timeout = 1 # seconds
+def isAlive(self):
+return (self._registered or self.pid is not None) and self.returncode
is None
+
@property
def pid(self):
-return self._proc.pid
+return None if self._proc is None else self._proc.pid
def _poll(self):
# Simply rely on _async_waitpid_cb to set the returncode.
--
2.41.0
[gentoo-portage-dev] [PATCH 1/9] anydbm: Pickle support for multiprocessing spawn
The egencache usage in ResolverPlayground that was used to trigger
bug 924319 triggered a pickling error for AuxdbTestCase.test_anydbm
with the multiprocessing spawn start method, so fix the anydbm
cache module to omit the unpicklable database object from pickled
state, and regenerate it after unpickling.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico
---
lib/portage/cache/anydbm.py | 17 -
lib/portage/tests/dbapi/test_auxdb.py | 4 +---
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/lib/portage/cache/anydbm.py b/lib/portage/cache/anydbm.py
index 94a270a483..ad7042ae41 100644
--- a/lib/portage/cache/anydbm.py
+++ b/lib/portage/cache/anydbm.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2020 Gentoo Authors
+# Copyright 2005-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# Author(s): Brian Harring (ferri...@gentoo.org)
@@ -67,6 +67,21 @@ class database(fs_template.FsBased):
raise cache_errors.InitializationError(self.__class__, e)
self._ensure_access(self._db_path)
+def __getstate__(self):
+state = self.__dict__.copy()
+# These attributes are not picklable, so they are automatically
+# regenerated after unpickling.
+state["_database__db"] = None
+return state
+
+def __setstate__(self, state):
+self.__dict__.update(state)
+mode = "w"
+if dbm.whichdb(self._db_path) in ("dbm.gnu", "gdbm"):
+# Allow multiple concurrent writers (see bug #53607).
+mode += "u"
+self.__db = dbm.open(self._db_path, mode, self._perms)
+
def iteritems(self):
# dbm doesn't implement items()
for k in self.__db.keys():
diff --git a/lib/portage/tests/dbapi/test_auxdb.py
b/lib/portage/tests/dbapi/test_auxdb.py
index 0de0123a5f..aac6ce361c 100644
--- a/lib/portage/tests/dbapi/test_auxdb.py
+++ b/lib/portage/tests/dbapi/test_auxdb.py
@@ -16,9 +16,7 @@ class AuxdbTestCase(TestCase):
from portage.cache.anydbm import database
except ImportError:
self.skipTest("dbm import failed")
-self._test_mod(
-"portage.cache.anydbm.database", multiproc=False, picklable=False
-)
+self._test_mod("portage.cache.anydbm.database", multiproc=False,
picklable=True)
def test_flat_hash_md5(self):
self._test_mod("portage.cache.flat_hash.md5_database")
--
2.41.0
[gentoo-portage-dev] [PATCH 0/9] EbuildPhase/EbuildMetadataPhase: async check_locale
Change config.environ() check_locale calls to async_check_locale calls in the EbuildPhase/EbuildMetadataPhase _async_start method in order to eliminate synchronous waiting for child processes in the main event loop thread. Note that this series of changes causes access to the portdbapi doebuild_settings attribute to no longer be serialized via the EbuildMetadataPhase _start_method. As a result, exclusive access to config instances needs to be guaranteed in some other way to avoid triggering problems (see bug 924319), such as by maintaining a config pool or by serializing access via an asyncio.Lock instance. This series can also be reviewed at https://github.com/gentoo/portage/pull/1267. Bug: https://bugs.gentoo.org/923841 Bug: https://bugs.gentoo.org/924319 Signed-off-by: Zac Medico Zac Medico (9): anydbm: Pickle support for multiprocessing spawn EbuildMetadataPhase: Add deallocate_config future MetadataRegen: Use EbuildMetadataPhase deallocate_config ResolverPlayground: Use egencache to create manifests asyncio: Wrap asyncio.Lock for python 3.9 compat async_aux_get: Use EbuildMetadataPhase deallocate_config future actions: disable pytest-xdist for spawn start-method (workers crash) EbuildMetadataPhase: Migrate to _async_start EbuildPhase: async_check_locale .github/workflows/ci.yml | 5 +- lib/_emerge/EbuildMetadataPhase.py| 37 - lib/_emerge/EbuildPhase.py| 28 +++- lib/_emerge/MetadataRegen.py | 16 ++- lib/_emerge/SubProcess.py | 5 +- lib/_emerge/depgraph.py | 11 ++ lib/portage/_emirrordist/FetchIterator.py | 10 +- lib/portage/cache/anydbm.py | 17 ++- lib/portage/dbapi/porttree.py | 129 -- lib/portage/package/ebuild/config.py | 26 ++-- lib/portage/tests/dbapi/test_auxdb.py | 4 +- lib/portage/tests/dbapi/test_portdb_cache.py | 4 +- .../tests/resolver/ResolverPlayground.py | 38 +++--- lib/portage/tests/update/test_move_ent.py | 3 + lib/portage/util/futures/_asyncio/__init__.py | 26 lib/portage/util/locale.py| 28 ++-- 16 files changed, 289 insertions(+), 98 deletions(-) -- 2.41.0
[gentoo-portage-dev] [PATCH v2] process.spawn: Add returnproc parameter
In order to migrate away from unsafe os.fork() usage in threaded
processes (https://github.com/python/cpython/issues/84559), add a
returnproc parameter that is similar to returnpid, which causes
spawn to return a single Process object instead of a list of pids.
The Process API is a subset of asyncio.subprocess.Process. The
returnproc parameter conflicts with the logfile parameter, since
the caller is expected to use the fd_pipes parameter to implement
logging (this was also true for the returnpid parameter).
In the future, spawn will return objects of a different type but
with a compatible interface to Process, in order to encapsulate
implementation-dependent objects like multiprocessing.Process which
are designed to manage the process lifecycle and need to persist
until it exits.
Trigger a UserWarning when the returnpid parameter is used, in
order to encourage migration to returnproc (do not use
DeprecationWarning since it is hidden by default). This warning
will be temporarily suppressed for portage internals, until they
finish migrating to returnproc. There are probably very few if
any external consumers of spawn with the returnpid parameter,
so it seems safe to move quickly with this deprecation.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico
---
[PATCH v2] Make returnproc return a singlar Process (for
comparison, returnpid never returned a list container more than
a single pid). Also, returnproc explicitly conflicts with the
logfile parameter, since callers are expected to use the fd_pipes
parameter to implement logging (this was also true for returnpid).
lib/portage/process.py| 84 +++
lib/portage/tests/process/meson.build | 1 +
.../tests/process/test_spawn_returnproc.py| 39 +
3 files changed, 124 insertions(+)
create mode 100644 lib/portage/tests/process/test_spawn_returnproc.py
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 71750a715f..6ec52efc4a 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -15,6 +15,7 @@ import subprocess
import sys
import traceback
import os as _os
+import warnings
from dataclasses import dataclass
from functools import lru_cache
@@ -27,6 +28,7 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util:dump_traceback,writemsg,writemsg_level",
)
@@ -277,12 +279,78 @@ def calc_env_stats(env) -> EnvStats:
env_too_large_warnings = 0
+class Process:
+"""
+An object that wraps OS processes created by spawn.
+In the future, spawn will return objects of a different type
+but with a compatible interface to this class, in order
+to encapsulate implementation-dependent objects like
+multiprocessing.Process which are designed to manage
+the process lifecycle and need to persist until it exits.
+"""
+
+def __init__(self, pid):
+self.pid = pid
+self.returncode = None
+self._exit_waiters = []
+
+def __repr__(self):
+return f"<{self.__class__.__name__} {self.pid}>"
+
+async def wait(self):
+"""
+Wait for the child process to terminate.
+
+Set and return the returncode attribute.
+"""
+if self.returncode is not None:
+return self.returncode
+
+loop = global_event_loop()
+if not self._exit_waiters:
+loop._asyncio_child_watcher.add_child_handler(self.pid,
self._child_handler)
+waiter = loop.create_future()
+self._exit_waiters.append(waiter)
+return await waiter
+
+def _child_handler(self, pid, returncode):
+if pid != self.pid:
+raise AssertionError(f"expected pid {self.pid}, got {pid}")
+self.returncode = returncode
+
+for waiter in self._exit_waiters:
+if not waiter.cancelled():
+waiter.set_result(returncode)
+self._exit_waiters = None
+
+def send_signal(self, sig):
+"""Send a signal to the process."""
+if self.returncode is not None:
+# Skip signalling a process that we know has already died.
+return
+
+try:
+os.kill(self.pid, sig)
+except ProcessLookupError:
+# Suppress the race condition error; bpo-40550.
+pass
+
+def terminate(self):
+"""Terminate the process with SIGTERM"""
+self.send_signal(signal.SIGTERM)
+
+def kill(self):
+"""Kill the process with SIGKILL"""
+self.send_signal(signal.SIGKILL)
+
+
def spawn(
mycommand,
env=None,
opt_name=None,
fd_pipes=None,
returnpid=False,
+returnproc=False,
uid=None,
gid=None,
groups
[gentoo-portage-dev] [PATCH] process.spawn: Add returnproc parameter
In order to migrate away from unsafe os.fork() usage in threaded
processes (https://github.com/python/cpython/issues/84559), add a
returnproc parameter that is similar to returnpid, which causes
spawn to return Process objects instead of pids. The Process
API is a subset of asyncio.subprocess.Process.
In the future, spawn will return objects of a different type but
with a compatible interface to Process, in order to encapsulate
implementation-dependent objects like multiprocessing.Process which
are designed to manage the process lifecycle and need to persist
until it exits.
Trigger a UserWarning when the returnpid parameter is used, in
order to encourage migration to returnproc (do not use
DeprecationWarning since it is hidden by default). This warning
will be temporarily suppressed for portage internals, until they
finish migrating to returnproc. There are probably very few if
any external consumers of spawn with the returnpid parameter,
so it seems safe to move quickly with this deprecation.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico
---
lib/portage/process.py| 79 +++
lib/portage/tests/process/meson.build | 1 +
.../tests/process/test_spawn_returnproc.py| 39 +
3 files changed, 119 insertions(+)
create mode 100644 lib/portage/tests/process/test_spawn_returnproc.py
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 71750a715f..74953b3e22 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -15,6 +15,7 @@ import subprocess
import sys
import traceback
import os as _os
+import warnings
from dataclasses import dataclass
from functools import lru_cache
@@ -27,6 +28,7 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util:dump_traceback,writemsg,writemsg_level",
)
@@ -277,12 +279,78 @@ def calc_env_stats(env) -> EnvStats:
env_too_large_warnings = 0
+class Process:
+"""
+An object that wraps OS processes created by spawn.
+In the future, spawn will return objects of a different type
+but with a compatible interface to this class, in order
+to encapsulate implementation-dependent objects like
+multiprocessing.Process which are designed to manage
+the process lifecycle and need to persist until it exits.
+"""
+
+def __init__(self, pid):
+self.pid = pid
+self.returncode = None
+self._exit_waiters = []
+
+def __repr__(self):
+return f"<{self.__class__.__name__} {self.pid}>"
+
+async def wait(self):
+"""
+Wait for the child process to terminate.
+
+Set and return the returncode attribute.
+"""
+if self.returncode is not None:
+return self.returncode
+
+loop = global_event_loop()
+if not self._exit_waiters:
+loop._asyncio_child_watcher.add_child_handler(self.pid,
self._child_handler)
+waiter = loop.create_future()
+self._exit_waiters.append(waiter)
+return await waiter
+
+def _child_handler(self, pid, returncode):
+if pid != self.pid:
+raise AssertionError(f"expected pid {self.pid}, got {pid}")
+self.returncode = returncode
+
+for waiter in self._exit_waiters:
+if not waiter.cancelled():
+waiter.set_result(returncode)
+self._exit_waiters = None
+
+def send_signal(self, sig):
+"""Send a signal to the process."""
+if self.returncode is not None:
+# Skip signalling a process that we know has already died.
+return
+
+try:
+os.kill(self.pid, sig)
+except ProcessLookupError:
+# Suppress the race condition error; bpo-40550.
+pass
+
+def terminate(self):
+"""Terminate the process with SIGTERM"""
+self.send_signal(signal.SIGTERM)
+
+def kill(self):
+"""Kill the process with SIGKILL"""
+self.send_signal(signal.SIGKILL)
+
+
def spawn(
mycommand,
env=None,
opt_name=None,
fd_pipes=None,
returnpid=False,
+returnproc=False,
uid=None,
gid=None,
groups=None,
@@ -315,6 +383,9 @@ def spawn(
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will
clean them.
@type returnpid: Boolean
+@param returnproc: Return the Process objects for a successful spawn.
+NOTE: This requires the caller clean up all the PIDs, otherwise spawn will
clean them.
+@type returnproc: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
Re: [gentoo-portage-dev] [PATCH 2/2] Add caching to _slot_operator_check_reverse_dependencies
On 11/24/22 19:36, Pin-yen Lin wrote: Add lru_cache to speed up the running time of "Calculating dependencies". In a ChromeOS use case, this patch decreases the running time from 311s to 197s with almost no memory usage increase. Signed-off-by: Pin-yen Lin --- lib/_emerge/depgraph.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index ce6cabcc1..9649bb2a8 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -2240,6 +2240,7 @@ class depgraph: return None +@functools.lru_cache(maxsize=100) def _slot_operator_check_reverse_dependencies( self, existing_pkg, candidate_pkg, replacement_parent=None ): Merged. Thank you! https://gitweb.gentoo.org/proj/portage.git/commit/?id=0c42cc962e1926ecbdc83d903a2804f9e037f2a9 https://gitweb.gentoo.org/proj/portage.git/commit/?id=839ab46be1777e5886da28b98b53a462b992c5bf -- Thanks, Zac
Re: [gentoo-portage-dev] usage of /bin/bash in shebangs
On 7/24/22 23:17, Fabian Groffen wrote: On 24-07-2022 13:58:31 -0700, Zac Medico wrote: On 7/24/22 12:29, Fabian Groffen wrote: Hi, Quick question, I noticed that portage uses /bin/bash hardcoded in shebang of scripts, while it uses /usr/bin/env python for python executable files. Is there anything against using /usr/bin/env bash for shell scripts? Changing this would help for Prefix. I can't think of any reason not to do this. Do you want to see a patch, or is it OK to push this myself? It's OK to just push this yourself. Thanks! -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
Re: [gentoo-portage-dev] usage of /bin/bash in shebangs
On 7/24/22 12:29, Fabian Groffen wrote: Hi, Quick question, I noticed that portage uses /bin/bash hardcoded in shebang of scripts, while it uses /usr/bin/env python for python executable files. Is there anything against using /usr/bin/env bash for shell scripts? Changing this would help for Prefix. I can't think of any reason not to do this. -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
Re: [gentoo-portage-dev] Normaliser function for distfiles
On 5/16/22 10:37, Markus Walter wrote: Hello all, is it possible to do the following: after fetching a distfile portage runs an external normaliser program specified in an ebuild before checking the hash? My use case is the following: I would like to improve the gs-elpa program and provide a precomputed overlay for melpa. However the melpa distfiles are rebuilt everyday and cause checksum failures. However the only thing changing are the timestamps. Hence if a normaliser program could simply set all timestamps to some predefined value (say 1.1.1970) then this problem should vanish. Thanks in advance Markus The only usable hook that we currently have for this is FETCHCOMMAND and RESUMCOMMAND in make.conf. You can replace them with a script that does the normal thing and then sets the timestamp. The default values are found in /usr/share/portage/config/make.globals. -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] MergeProcess: propagate mtimedb["ldpath"] to parent process (bug 836375)
Use an instance of multiprocessing.Pipe to propagate mtimedb["ldpath"]
from the MergeProcess child process to the parent process. This fixes
env_update calls to avoid unnecessary regeneration of ld.so.cache in
cases where mtimedb["ldpath"] has not changed since the last call to
env_update.
Bug: https://bugs.gentoo.org/836375
---
lib/portage/dbapi/_MergeProcess.py | 18 ++
lib/portage/dbapi/vartree.py | 10 ++
2 files changed, 28 insertions(+)
diff --git a/lib/portage/dbapi/_MergeProcess.py
b/lib/portage/dbapi/_MergeProcess.py
index db3f3b105..667a5bf20 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -2,6 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
import io
+import multiprocessing
import platform
import fcntl
@@ -38,6 +39,7 @@ class MergeProcess(ForkProcess):
"_dblink",
"_elog_keys",
"_locked_vdb",
+"_mtime_reader",
)
def _start(self):
@@ -113,6 +115,15 @@ class MergeProcess(ForkProcess):
self._elog_reader_fd = None
return False
+def _mtime_handler(self):
+try:
+mtimes = self._mtime_reader.recv()
+except EOFError:
+pass
+else:
+self.prev_mtimes.clear()
+self.prev_mtimes.update(mtimes)
+
def _spawn(self, args, fd_pipes, **kwargs):
"""
Extend the superclass _spawn method to perform some pre-fork and
@@ -127,6 +138,11 @@ class MergeProcess(ForkProcess):
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
)
+mtime_reader, mtime_writer = multiprocessing.Pipe(duplex=False)
+fd_pipes[mtime_writer.fileno()] = mtime_writer.fileno()
+self.scheduler.add_reader(mtime_reader.fileno(), self._mtime_handler)
+self._mtime_reader = mtime_reader
+
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
@@ -142,6 +158,7 @@ class MergeProcess(ForkProcess):
vartree=self.vartree,
blockers=blockers,
pipe=elog_writer_fd,
+mtime_pipe=mtime_writer,
)
fd_pipes[elog_writer_fd] = elog_writer_fd
self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler)
@@ -160,6 +177,7 @@ class MergeProcess(ForkProcess):
self._elog_reader_fd = elog_reader_fd
pids = super(MergeProcess, self)._spawn(args, fd_pipes, **kwargs)
os.close(elog_writer_fd)
+mtime_writer.close()
self._buf = ""
self._elog_keys = set()
# Discard messages which will be collected by the subprocess,
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 602913862..a95d60691 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1806,6 +1806,7 @@ class dblink:
blockers=None,
scheduler=None,
pipe=None,
+mtime_pipe=None,
):
"""
Creates a DBlink object for a given CPV.
@@ -1862,6 +1863,7 @@ class dblink:
self._device_path_map = {}
self._hardlink_merge_map = {}
self._hash_key = (self._eroot, self.mycpv)
+self._mtime_pipe = mtime_pipe
self._protect_obj = None
self._pipe = pipe
self._postinst_failure = False
@@ -2618,6 +2620,7 @@ class dblink:
writemsg_level=self._display_merge,
vardbapi=self.vartree.dbapi,
)
+self._send_mtimes(ldpath_mtimes)
unmerge_with_replacement = preserve_paths is not None
if not unmerge_with_replacement:
@@ -4243,6 +4246,12 @@ class dblink:
def _emerge_log(self, msg):
emergelog(False, msg)
+def _send_mtimes(self, mtimes):
+if self._mtime_pipe is None:
+return
+
+self._mtime_pipe.send(mtimes)
+
def treewalk(
self,
srcroot,
@@ -5274,6 +5283,7 @@ class dblink:
writemsg_level=self._display_merge,
vardbapi=self.vartree.dbapi,
)
+self._send_mtimes(prev_mtimes)
# For gcc upgrades, preserved libs have to be removed after the
# the library path has been updated.
--
2.35.1
Re: [gentoo-portage-dev] The build system (and install layout) of Portage
On 3/17/22 10:22, Michał Górny wrote: Hi, everyone. You've probably had the opportunity to hear that a lot has changed in Python packaging since Portage's setup.py was written in 2014. There were some minor changes to keep it working since but it's time to reconsider. Long story short, distutils is strongly deprecated, setuptools deprecated most of the customizations (and we're relying heavily on customizations), PEP 517 doesn't cover our use cases exactly... and it's quite likely that sooner or later our build system will fall apart. On top of that, setuptools is going through a stage of "let's vendor a new dependency every week", so it doesn't look like a feasible long-term solution. A large part of the problem is that Portage is heavily relying on non-Pythonic idioms for installing stuff. I wonder if we can rely less on the deprecated customizations somehow. For example, the venv_data_files function in setup.py succeeds in installing a bunch of files in custom locations, and maybe we can rely more on that approach (use it not only for venv installations). -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] Revert "dep_zapdeps: avoid new slots when appropriate (bug 828136)"
Revert the change from bug 828136, since it prevents solving
of some blockers unless --update and --deep are specified as
reported in bug 833014.
Bug: https://bugs.gentoo.org/833014
Reverts: a7289ac0eaaa0d435bf6d9bfb2724a6b39adcbee
Signed-off-by: Zac Medico
---
lib/portage/dep/dep_check.py | 6 +-
.../tests/resolver/test_installkernel.py | 20 +--
.../resolver/test_unecessary_slot_upgrade.py | 11 --
3 files changed, 2 insertions(+), 35 deletions(-)
diff --git a/lib/portage/dep/dep_check.py b/lib/portage/dep/dep_check.py
index 8ca4c0b9d..9fccda08b 100644
--- a/lib/portage/dep/dep_check.py
+++ b/lib/portage/dep/dep_check.py
@@ -376,7 +376,6 @@ def dep_zapdeps(
# c) contains masked installed packages
# d) is the first item
-no_new_slots = []
preferred_in_graph = []
preferred_installed = preferred_in_graph
preferred_any_slot = preferred_in_graph
@@ -392,7 +391,6 @@ def dep_zapdeps(
# unsat_use_* must come after preferred_non_installed
# for correct ordering in cases like || ( foo[a] foo[b] ).
choice_bins = (
-no_new_slots,
preferred_in_graph,
preferred_non_installed,
unsat_use_in_graph,
@@ -691,9 +689,7 @@ def dep_zapdeps(
other.append(this_choice)
else:
if all_use_satisfied:
-if new_slot_count == 0 and not want_update:
-no_new_slots.append(this_choice)
-elif all_in_graph:
+if all_in_graph:
preferred_in_graph.append(this_choice)
elif all_installed:
if all_installed_slots:
diff --git a/lib/portage/tests/resolver/test_installkernel.py
b/lib/portage/tests/resolver/test_installkernel.py
index b73bbe5bb..5909b53aa 100644
--- a/lib/portage/tests/resolver/test_installkernel.py
+++ b/lib/portage/tests/resolver/test_installkernel.py
@@ -58,25 +58,8 @@ class InstallKernelTestCase(TestCase):
),
],
),
-# Demonstrate bug 833014, where the calculation fails unless
+# Test bug 833014, where the calculation failed unless
# --update and --deep are specified.
-ResolverPlaygroundTestCase(
-[
-"sys-kernel/installkernel-systemd-boot",
-"sys-kernel/gentoo-kernel-bin",
-],
-ambiguous_merge_order=True,
-success=False,
-mergelist=[
-"sys-kernel/installkernel-systemd-boot-1",
-"sys-kernel/gentoo-kernel-bin-5.15.23",
-"virtual/dist-kernel-5.15.23",
-(
-"!sys-kernel/installkernel-gentoo",
-"!sys-kernel/installkernel-systemd-boot",
-),
-],
-),
ResolverPlaygroundTestCase(
[
"sys-kernel/installkernel-systemd-boot",
@@ -84,7 +67,6 @@ class InstallKernelTestCase(TestCase):
],
ambiguous_merge_order=True,
success=True,
-options={"--deep": True, "--update": True},
mergelist=[
"virtual/dist-kernel-5.15.23",
"sys-kernel/installkernel-systemd-boot-1",
diff --git a/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
b/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
index f8b8b346a..a89ebdb67 100644
--- a/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
+++ b/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
@@ -26,13 +26,6 @@ class UnnecessarySlotrUpgradeTestCase(TestCase):
test_cases = (
# Test bug 828136, where an unnecessary python slot upgrade
# was triggered.
-ResolverPlaygroundTestCase(
-[
-"app-misc/a",
-],
-success=True,
-mergelist=("app-misc/a-1",),
-),
ResolverPlaygroundTestCase(
[
"app-misc/a",
@@ -42,10 +35,6 @@ class UnnecessarySlotrUpgradeTestCase(TestCase):
"dev-lang/python-3.10",
"app-misc/a-1",
),
-options={
-"--deep": True,
-"--update": True,
-},
),
)
--
2.34.1
Re: [gentoo-portage-dev] [PATCH 4/4] portage.eapi: use functools @lru_cache decorator instead of custom implementation
On 2/26/22 10:04, Zac Medico wrote: On 2/23/22 20:14, Matt Turner wrote: From: "Wolfgang E. Sanyer" Reviewed-by: Matt Turner Signed-off-by: Wolfgang E. Sanyer --- Â lib/portage/eapi.py | 155 Â 1 file changed, 72 insertions(+), 83 deletions(-) diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py index 56e64620a..efcc6c2a0 100644 --- a/lib/portage/eapi.py +++ b/lib/portage/eapi.py @@ -2,12 +2,10 @@ Â # Distributed under the terms of the GNU General Public License v2 Â import collections -import operator -import types - -from portage import eapi_is_supported +from functools import lru_cache +@lru_cache(None) Â def eapi_has_iuse_defaults(eapi): Â if eapi is None: Â return True @@ -15,6 +13,7 @@ def eapi_has_iuse_defaults(eapi): Â return eapi != "0" I think this patch misses the point of the original caching mechanism. It doesn't make sense to cache results of the individual eapi_* functions if they no longer contribute to the _eapi_attrs cache. To clarify, the only reason that the eapi_* functions were cached was so that they would trigger population of the _eapi_attrs cache. In the absence of this_eapi_attrs cache population feature, I doubt that it's very useful to put the lru_cache on the indivdual eapi_* functions. -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH 4/4] portage.eapi: use functools @lru_cache decorator instead of custom implementation
On 2/23/22 20:14, Matt Turner wrote: From: "Wolfgang E. Sanyer" Reviewed-by: Matt Turner Signed-off-by: Wolfgang E. Sanyer --- lib/portage/eapi.py | 155 1 file changed, 72 insertions(+), 83 deletions(-) diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py index 56e64620a..efcc6c2a0 100644 --- a/lib/portage/eapi.py +++ b/lib/portage/eapi.py @@ -2,12 +2,10 @@ # Distributed under the terms of the GNU General Public License v2 import collections -import operator -import types - -from portage import eapi_is_supported +from functools import lru_cache +@lru_cache(None) def eapi_has_iuse_defaults(eapi): if eapi is None: return True @@ -15,6 +13,7 @@ def eapi_has_iuse_defaults(eapi): return eapi != "0" I think this patch misses the point of the original caching mechanism. It doesn't make sense to cache results of the individual eapi_* functions if they no longer contribute to the _eapi_attrs cache. -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] repoman: Remove http compatibility code for metadata DTD
On 1/25/22 08:44, Ulrich Müller wrote:
Commit 3950d76df says: "The http:// compat can be removed once the
Gentoo repository is updated to use https:// everywhere."
Bug: https://bugs.gentoo.org/552720
Signed-off-by: Ulrich Müller
---
repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py
b/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py
index b4e0ee933..0fb97a0df 100644
--- a/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py
@@ -127,7 +127,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
)
else:
doctype_system = _metadata_xml.docinfo.system_url
-if doctype_system.replace("http://;, "https://;) !=
metadata_dtd_uri:
+if doctype_system != metadata_dtd_uri:
if doctype_system is None:
system_problem = "but it is undefined"
else:
Looks good. Please merge.
--
Thanks,
Zac
OpenPGP_signature
Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] file_copy: handle zero bytes copied by copy_file_range (bug 828844)
When copy_file_range copied zero bytes, fall back to sendfile,
so that we don't call copy_file_range in an infinite loop.
Bug: https://bugs.gentoo.org/828844
Tested-by: John Helmert III
Signed-off-by: Zac Medico
---
src/portage_util_file_copy_reflink_linux.c | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/portage_util_file_copy_reflink_linux.c
b/src/portage_util_file_copy_reflink_linux.c
index c6affe57a..b00b57952 100644
--- a/src/portage_util_file_copy_reflink_linux.c
+++ b/src/portage_util_file_copy_reflink_linux.c
@@ -261,13 +261,14 @@ _reflink_linux_file_copy(PyObject *self, PyObject *args)
_out,
len);
-if (copyfunc_ret < 0) {
+if (copyfunc_ret <= 0) {
error = errno;
-if ((errno == EXDEV || errno == ENOSYS || errno ==
EOPNOTSUPP) &&
+if ((errno == EXDEV || errno == ENOSYS || errno ==
EOPNOTSUPP || copyfunc_ret == 0) &&
copyfunc == cfr_wrapper) {
/* Use sendfile instead of copy_file_range for
* cross-device copies, or when the copy_file_range
- * syscall is not available (less than Linux 4.5).
+ * syscall is not available (less than Linux 4.5),
+ * or when copy_file_range copies zero bytes.
*/
error = 0;
copyfunc = sf_wrapper;
--
2.32.0
[gentoo-portage-dev] [PATCH] dep_zapdeps: avoid new slots when appropriate (bug 828136)
Place choices that do not pull in new slots into a preferred
choice bin, so that they will not be mixed with choices that
contain unnecessary upgrades. This fixes the included test
case so that an unnecessary new python slot is not pulled in.
Bug: https://bugs.gentoo.org/828136
Signed-off-by: Zac Medico
---
lib/portage/dep/dep_check.py | 6 +-
.../resolver/test_unecessary_slot_upgrade.py | 62 +++
2 files changed, 67 insertions(+), 1 deletion(-)
create mode 100644 lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
diff --git a/lib/portage/dep/dep_check.py b/lib/portage/dep/dep_check.py
index 9fccda08b..8ca4c0b9d 100644
--- a/lib/portage/dep/dep_check.py
+++ b/lib/portage/dep/dep_check.py
@@ -376,6 +376,7 @@ def dep_zapdeps(
# c) contains masked installed packages
# d) is the first item
+no_new_slots = []
preferred_in_graph = []
preferred_installed = preferred_in_graph
preferred_any_slot = preferred_in_graph
@@ -391,6 +392,7 @@ def dep_zapdeps(
# unsat_use_* must come after preferred_non_installed
# for correct ordering in cases like || ( foo[a] foo[b] ).
choice_bins = (
+no_new_slots,
preferred_in_graph,
preferred_non_installed,
unsat_use_in_graph,
@@ -689,7 +691,9 @@ def dep_zapdeps(
other.append(this_choice)
else:
if all_use_satisfied:
-if all_in_graph:
+if new_slot_count == 0 and not want_update:
+no_new_slots.append(this_choice)
+elif all_in_graph:
preferred_in_graph.append(this_choice)
elif all_installed:
if all_installed_slots:
diff --git a/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
b/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
new file mode 100644
index 0..f8b8b346a
--- /dev/null
+++ b/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
@@ -0,0 +1,62 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+ResolverPlayground,
+ResolverPlaygroundTestCase,
+)
+
+
+class UnnecessarySlotrUpgradeTestCase(TestCase):
+def testUnnecessarySlotUpgrade(self):
+ebuilds = {
+"app-misc/a-1": {
+"EAPI": "8",
+"RDEPEND": "|| ( dev-lang/python:3.10 dev-lang/python:3.9 ) ||
( dev-lang/python:3.10 dev-lang/python:3.9 )",
+},
+"dev-lang/python-3.9": {"SLOT": "3.9"},
+"dev-lang/python-3.10": {"SLOT": "3.10"},
+}
+
+installed = {
+"dev-lang/python-3.9": {"SLOT": "3.9"},
+}
+
+test_cases = (
+# Test bug 828136, where an unnecessary python slot upgrade
+# was triggered.
+ResolverPlaygroundTestCase(
+[
+"app-misc/a",
+],
+success=True,
+mergelist=("app-misc/a-1",),
+),
+ResolverPlaygroundTestCase(
+[
+"app-misc/a",
+],
+success=True,
+mergelist=(
+"dev-lang/python-3.10",
+"app-misc/a-1",
+),
+options={
+"--deep": True,
+"--update": True,
+},
+),
+)
+
+playground = ResolverPlayground(
+debug=False, ebuilds=ebuilds, installed=installed
+)
+
+try:
+for test_case in test_cases:
+playground.run_TestCase(test_case)
+self.assertEqual(test_case.test_success, True,
test_case.fail_msg)
+finally:
+playground.debug = False
+playground.cleanup()
--
2.32.0
[gentoo-portage-dev] [PATCH] emerge: Default enable soname dependencies (bug 687956)
Default emerge --ignore-soname-deps=n, in order to enable
soname dependencies by default. As always, soname dependencies
remain inapplicable in the absence of the --usepkgonly option
(or --getbinpkgonly). Therefore, this change only affects
commands that specify --usepkgonly or --getbinpkgonly.
Bug: https://bugs.gentoo.org/687956
Signed-off-by: Zac Medico
---
lib/_emerge/create_depgraph_params.py | 2 +-
man/emerge.1 | 7 ---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/lib/_emerge/create_depgraph_params.py
b/lib/_emerge/create_depgraph_params.py
index 11c3e3736..95c4c2035 100644
--- a/lib/_emerge/create_depgraph_params.py
+++ b/lib/_emerge/create_depgraph_params.py
@@ -104,7 +104,7 @@ def create_depgraph_params(myopts, myaction):
if ignore_built_slot_operator_deps is not None:
myparams["ignore_built_slot_operator_deps"] =
ignore_built_slot_operator_deps
-myparams["ignore_soname_deps"] = myopts.get("--ignore-soname-deps", "y")
+myparams["ignore_soname_deps"] = myopts.get("--ignore-soname-deps", "n")
dynamic_deps = myopts.get("--dynamic-deps", "y") != "n" and "--nodeps" not
in myopts
if dynamic_deps:
diff --git a/man/emerge.1 b/man/emerge.1
index 8f6d12925..ff565b46f 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -639,9 +639,10 @@ supported beginning with \fBEAPI 5\fR.
.TP
.BR "\-\-ignore\-soname\-deps < y | n >"
Ignore the soname dependencies of binary and installed packages. This
-option is enabled by default, since soname dependencies are relatively
-new, and the required metadata is not guaranteed to exist for binary and
-installed packages built with older versions of portage. Also, soname
+option may be useful when working with binary or installed packages
+that lack appropriate soname dependency metadata because they were built
+with a package manager that does not support soname dependencies (perhaps
+an older version of portage). Soname
dependencies will be automatically ignored for dependency calculations
that can pull unbuilt ebuilds into the dependency graph, since unbuilt
ebuilds do not have any soname dependency metadata, making it impossible
--
2.32.0
Re: [gentoo-portage-dev] [PATCH] Install example repo.postsync.d script into sharedir
On 11/4/21 02:50, Daniel Cordero wrote:
The sysconfdir is for host specific configuration files, and this
example script makes no host specific change (it is not enabled by
default).
Install the script under portage's sharedir, from where administrators
can copy it into sysconfdir, if needed.
Signed-off-by: Daniel Cordero
---
setup.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/setup.py b/setup.py
index 549fff650..fe0265c64 100755
--- a/setup.py
+++ b/setup.py
@@ -833,12 +833,12 @@ setup(
["$portage_setsdir", ["cnf/sets/portage.conf"]],
["$docdir", ["NEWS", "RELEASE-NOTES"]],
["$portage_base/bin", ["bin/deprecated-path"]],
-["$sysconfdir/portage/repo.postsync.d",
["cnf/repo.postsync.d/example"]],
+["$portage_confdir/repo.postsync.d",
["cnf/repo.postsync.d/example"]],
],
[
("etc", "cnf", ("etc-update.conf", "dispatch-conf.conf")),
("etc/logrotate.d", "cnf/logrotate.d", ("elog-save-summary",)),
-("etc/portage/repo.postsync.d", "cnf/repo.postsync.d",
("example",)),
+("share/portage/config/repo.postsync.d", "cnf/repo.postsync.d",
("example",)),
(
"share/portage/config",
"cnf",
Merged, thanks!
https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7dc908a99e564de6f10174b1489028d939b917f
--
Thanks,
Zac
[gentoo-portage-dev] Re: [PATCH] EbuildIpc.communicate: lockfile PermissionDenied retry
On 11/3/21 20:09, Zac Medico wrote: The lockfile function is expected to raise PermissionDenied if the (root) parent process holds the lock, so retry in this case. Bug: https://bugs.gentoo.org/468990 Signed-off-by: Zac Medico --- bin/ebuild-ipc.py | 11 ++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py index 4999c043a..6eaa658a2 100755 --- a/bin/ebuild-ipc.py +++ b/bin/ebuild-ipc.py @@ -158,7 +158,16 @@ class EbuildIpc: # Make locks quiet since unintended locking messages displayed on # stdout could corrupt the intended output of this program. portage.locks._quiet = True -lock_obj = portage.locks.lockfile(self.ipc_lock_file, unlinkfile=True) +# Acquire lock with PermissionDenied retry for bug #468990. +for _ in range(1000): +try: +lock_obj = portage.locks.lockfile(self.ipc_lock_file, unlinkfile=True) +except portage.exception.PermissionDenied: +time.sleep(0.1) +else: +break +else: +raise portage.exception.PermissionDenied(self.ipc_lock_file) try: return self._communicate(args) Withdrawn in favor of this lockfile permission race fix: https://archives.gentoo.org/gentoo-portage-dev/message/88916062415d9f692091dfb947f1bda2 -- Thanks, Zac OpenPGP_signature Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] EbuildIpcDaemon: fix lock permission race
Move ipc files to a .ipc subdirectory, with a setgid bit to
prevent a lockfile group permission race. The lockfile function
uses an appropriate open call with mode argument so that the
lockfile is created atomically with both group ownership and
group write bit.
Bug: https://bugs.gentoo.org/468990
Signed-off-by: Zac Medico
---
bin/ebuild-ipc.py| 6 +++---
bin/phase-functions.sh | 4 ++--
lib/_emerge/AbstractEbuildProcess.py | 4 ++--
lib/_emerge/EbuildIpcDaemon.py | 2 +-
lib/portage/package/ebuild/prepare_build_dirs.py | 6 ++
lib/portage/tests/ebuild/test_doebuild_spawn.py | 1 +
lib/portage/tests/ebuild/test_ipc_daemon.py | 6 +++---
7 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py
index 4999c043a..c24ba6f58 100755
--- a/bin/ebuild-ipc.py
+++ b/bin/ebuild-ipc.py
@@ -138,9 +138,9 @@ class EbuildIpc:
def __init__(self):
self.fifo_dir = os.environ["PORTAGE_BUILDDIR"]
-self.ipc_in_fifo = os.path.join(self.fifo_dir, ".ipc_in")
-self.ipc_out_fifo = os.path.join(self.fifo_dir, ".ipc_out")
-self.ipc_lock_file = os.path.join(self.fifo_dir, ".ipc_lock")
+self.ipc_in_fifo = os.path.join(self.fifo_dir, ".ipc", "in")
+self.ipc_out_fifo = os.path.join(self.fifo_dir, ".ipc", "out")
+self.ipc_lock_file = os.path.join(self.fifo_dir, ".ipc", "lock")
def _daemon_is_alive(self):
try:
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index d3221993d..5eb031805 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -291,10 +291,10 @@ __dyn_clean() {
rm -f
"$PORTAGE_BUILDDIR"/.{ebuild_changed,logid,pretended,setuped,unpacked,prepared}
\
"$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged,instprepped} \
"$PORTAGE_BUILDDIR"/.die_hooks \
- "$PORTAGE_BUILDDIR"/.ipc_{in,out,lock} \
"$PORTAGE_BUILDDIR"/.exit_status
- rm -rf "${PORTAGE_BUILDDIR}/build-info"
+ rm -rf "${PORTAGE_BUILDDIR}/build-info" \
+ "${PORTAGE_BUILDDIR}/.ipc"
rm -rf "${WORKDIR}"
rm -f "${PORTAGE_BUILDDIR}/files"
fi
diff --git a/lib/_emerge/AbstractEbuildProcess.py
b/lib/_emerge/AbstractEbuildProcess.py
index 1b4e7759f..6d89d40f0 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -249,8 +249,8 @@ class AbstractEbuildProcess(SpawnProcess):
def _init_ipc_fifos(self):
-input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in")
-output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".ipc_out")
+input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc",
"in")
+output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc",
"out")
for p in (input_fifo, output_fifo):
diff --git a/lib/_emerge/EbuildIpcDaemon.py b/lib/_emerge/EbuildIpcDaemon.py
index ee6fd7658..78594ff0a 100644
--- a/lib/_emerge/EbuildIpcDaemon.py
+++ b/lib/_emerge/EbuildIpcDaemon.py
@@ -81,7 +81,7 @@ class EbuildIpcDaemon(FifoIpcDaemon):
# write something to the pipe just before we close it, and in that
# case the write will be lost. Therefore, try for a non-blocking
# lock, and only re-open the pipe if the lock is acquired.
-lock_filename = os.path.join(os.path.dirname(self.input_fifo),
".ipc_lock")
+lock_filename = os.path.join(os.path.dirname(self.input_fifo),
"lock")
try:
lock_obj = lockfile(lock_filename, unlinkfile=True,
flags=os.O_NONBLOCK)
except TryAgain:
diff --git a/lib/portage/package/ebuild/prepare_build_dirs.py
b/lib/portage/package/ebuild/prepare_build_dirs.py
index 659198905..9d2474fd8 100644
--- a/lib/portage/package/ebuild/prepare_build_dirs.py
+++ b/lib/portage/package/ebuild/prepare_build_dirs.py
@@ -102,6 +102,12 @@ def prepare_build_dirs(myroot=None, settings=None,
cleanup=False):
apply_secpass_permissions(
mysettings[dir_key], uid=portage_uid, gid=portage_gid
)
+# The setgid bit prevents a lockfile group permission race for bug
#468990.
+ensure_dirs(
+os.path.join(mysettings["PORTAGE_BUILDDIR"], ".ipc"),
+gid=portage_gid,
+mode=0o2770,
+)
except PermissionDenied as e:
writemsg(_("Permission Denied: %s\n") %
[gentoo-portage-dev] [PATCH] fetch: enable resume for digestgen case
Enable resume for the digestgen case (no digests available), when
the temporary file exceeds PORTAGE_FETCH_RESUME_MIN_SIZE. This
fixes a case which caused the ebuild digest command to skip the
download and fail with a message like this:
!!! File b'/var/cache/distfiles/foo.tar.xz' doesn't exist, can't update Manifest
Bug: https://bugs.gentoo.org/821571
Signed-off-by: Zac Medico
---
lib/portage/package/ebuild/fetch.py | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py
b/lib/portage/package/ebuild/fetch.py
index 8c64362c2..2d3625800 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -1485,10 +1485,11 @@ def fetch(
except EnvironmentError:
pass
elif not orig_digests:
-# We don't have a digest, but the file exists. We must
-# assume that it is fully downloaded.
+# We don't have a digest, and the temporary file
exists.
if not force:
-continue
+# Try to resume this download when full
+# download has not been explicitly forced.
+fetched = 1
else:
if (
mydigests[myfile].get("size") is not None
--
2.32.0
Re: [gentoo-portage-dev] [PATCH] bin/estrip: avoid copying directories in FEATURES=installsources
On 7/17/21 12:59 PM, Sergei Trofimovich wrote:
> Initially problem is noticed on gcc-11 as a full ${WORKDIR} syncing
> into /usr/src/debug. It happens because `debug.sources` sometimes
> contains directory. For example on bash-5 it has:
>
> $ grep -zv '/<[^/>]*>$' debug.sources | LANG=C sort -z -u | sed -e
> 's/\x00/\n/g'
> bash-5.0/
> bash-5.0/alias.c
> ...
>
> This causes syncing object files, config.log, final binaries
> and other unexpected data. The change avoids syncking paths
> that end with '/'.
>
> Signed-off-by: Sergei Trofimovich
> ---
> bin/estrip | 3 +++
> 1 file changed, 3 insertions(+)
>
> diff --git a/bin/estrip b/bin/estrip
> index 7ef1ec35c..6cca0d04b 100755
> --- a/bin/estrip
> +++ b/bin/estrip
> @@ -464,7 +464,10 @@ if [[ -s ${tmpdir}/debug.sources ]] && \
> then
> __vecho "installsources: rsyncing source files"
> [[ -d ${D%/}/${prepstrip_sources_dir#/} ]] || mkdir -p
> "${D%/}/${prepstrip_sources_dir#/}"
> + # skip installation of ".../" (system headers? why inner slashes
> are forbidden?)
> + # skip syncing of ".../foo/" (complete directories)
> grep -zv '/<[^/>]*>$' "${tmpdir}"/debug.sources | \
> + grep -zv '/$' | \
> (cd "${WORKDIR}"; LANG=C sort -z -u | \
> rsync -tL0 --chmod=ugo-st,a+r,go-w,Da+x,Fa-x --files-from=-
> "${WORKDIR}/" "${D%/}/${prepstrip_sources_dir#/}/" )
>
>
Looks good. Merged with both grep calls combined via grep -e. Thanks!
https://gitweb.gentoo.org/proj/portage.git/commit/?id=e083c8bf20d8488d329e3dccd643c28429e6fe30
--
Thanks,
Zac
OpenPGP_signature
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] man/make.conf.5: remove mention of zlib USE flag
On 6/28/21 2:56 PM, Thymo van Beers wrote: > Both sys-devel/binutils and sys-devel/gdb are built with system zlib by > default for some time now. This commit removes the mention of USE=zlib to > avoid > confusion. > > Signed-off-by: Thymo van Beers > --- > man/make.conf.5 | 3 +-- > 1 file changed, 1 insertion(+), 2 deletions(-) > > diff --git a/man/make.conf.5 b/man/make.conf.5 > index 1c72109ad..db742fdb5 100644 > --- a/man/make.conf.5 > +++ b/man/make.conf.5 > @@ -378,8 +378,7 @@ redundant on\-the\-fly compression. The resulting file > will be called > .TP > .B compressdebug > Compress the debug sections in the split debug files with zlib to save > -space. Make sure you have built both binutils and gdb with USE=zlib > -support for this to work. See \fBsplitdebug\fR for general split debug > +space. See \fBsplitdebug\fR for general split debug > information (upon which this feature depends). > .TP > .B config\-protect\-if\-modified > Merged thanks! https://gitweb.gentoo.org/proj/portage.git/commit/?id=ee944c4fd76af4f7dffb756e9ed0303cb9606112 -- Thanks, Zac
[gentoo-portage-dev] Re: [PATCH v2] Add @unsatisfied-deps package set (bug 248026)
On 6/18/21 8:29 PM, Zac Medico wrote: > On 6/18/21 8:13 PM, Zac Medico wrote: >> On 6/18/21 6:01 PM, Zac Medico wrote: >>> If emerge --depclean fails to resolve any dependencies, then it will >>> now suggest emerge @unsatisfied-deps as the simplest possible >>> solution, and will also suggest to unmerge @unavailable where >>> appropriate at the end: >>> >>> $ emerge --depclean >>> >>> Calculating dependencies... done! >>> * Dependencies could not be completely resolved due to >>> * the following required packages not being installed: >>> * >>> * virtual/cdrtools pulled in by: >>> * app-cdr/cdrdao-1.2.4 >>> * >>> * Have you forgotten to resolve unsatisfied dependencies prior to >>> * depclean? The simplest possible command for this purpose is as >>> * follows: >>> * >>> * emerge @unsatisfied-deps >> >> It turns out that @unsatisfied-deps is often unsuitable here because it >> pulls in a bunch of @installed packages, when you really want to use >> @world as the source of truth. > > The underlying reason is the same as the reason that we've never used > @installed updates as a substitute for @world updates. It just doesn't > work, because @installed is polluted in comparison to @world. > My plan is it introduce an @unsatisfied-world set which is equivalent to @unsatisfied-deps but filters out any non @world packages. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] Re: [PATCH v2] Add @unsatisfied-deps package set (bug 248026)
On 6/18/21 8:13 PM, Zac Medico wrote: > On 6/18/21 6:01 PM, Zac Medico wrote: >> If emerge --depclean fails to resolve any dependencies, then it will >> now suggest emerge @unsatisfied-deps as the simplest possible >> solution, and will also suggest to unmerge @unavailable where >> appropriate at the end: >> >> $ emerge --depclean >> >> Calculating dependencies... done! >> * Dependencies could not be completely resolved due to >> * the following required packages not being installed: >> * >> * virtual/cdrtools pulled in by: >> * app-cdr/cdrdao-1.2.4 >> * >> * Have you forgotten to resolve unsatisfied dependencies prior to >> * depclean? The simplest possible command for this purpose is as >> * follows: >> * >> * emerge @unsatisfied-deps > > It turns out that @unsatisfied-deps is often unsuitable here because it > pulls in a bunch of @installed packages, when you really want to use > @world as the source of truth. The underlying reason is the same as the reason that we've never used @installed updates as a substitute for @world updates. It just doesn't work, because @installed is polluted in comparison to @world. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] Re: [PATCH v2] Add @unsatisfied-deps package set (bug 248026)
On 6/18/21 6:01 PM, Zac Medico wrote: > If emerge --depclean fails to resolve any dependencies, then it will > now suggest emerge @unsatisfied-deps as the simplest possible > solution, and will also suggest to unmerge @unavailable where > appropriate at the end: > > $ emerge --depclean > > Calculating dependencies... done! > * Dependencies could not be completely resolved due to > * the following required packages not being installed: > * > * virtual/cdrtools pulled in by: > * app-cdr/cdrdao-1.2.4 > * > * Have you forgotten to resolve unsatisfied dependencies prior to > * depclean? The simplest possible command for this purpose is as > * follows: > * > * emerge @unsatisfied-deps It turns out that @unsatisfied-deps is often unsuitable here because it pulls in a bunch of @installed packages, when you really want to use @world as the source of truth. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v2] Add @unsatisfied-deps package set (bug 248026)
If emerge --depclean fails to resolve any dependencies, then it will
now suggest emerge @unsatisfied-deps as the simplest possible
solution, and will also suggest to unmerge @unavailable where
appropriate at the end:
$ emerge --depclean
Calculating dependencies... done!
* Dependencies could not be completely resolved due to
* the following required packages not being installed:
*
* virtual/cdrtools pulled in by:
* app-cdr/cdrdao-1.2.4
*
* Have you forgotten to resolve unsatisfied dependencies prior to
* depclean? The simplest possible command for this purpose is as
* follows:
*
* emerge @unsatisfied-deps
*
* The most comprehensive possible update command is this:
*
* emerge --update --newuse --deep --with-bdeps=y @world
*
* Note that the --with-bdeps=y option is not required in many
* situations. Refer to the emerge manual page (run `man emerge`)
* for more information about --with-bdeps.
*
* Also, note that it may be necessary to manually uninstall
* packages that no longer exist in the repository, since it may not
* be possible to satisfy their dependencies. The simplest possible
* command for this purpose is as follows, but be careful to examine
* the resulting package list carefully:
*
* emerge --ask --unmerge @unavailable
*
Bug: https://bugs.gentoo.org/248026
Signed-off-by: Zac Medico
---
[PATCH v2] Update --depclean message to suggest @unsatisfied-deps
and unmerge @unavailable where appropriate.
cnf/sets/portage.conf | 5 +++
doc/config/sets.docbook | 7
lib/_emerge/actions.py| 19 +++--
lib/portage/_sets/__init__.py | 4 ++
lib/portage/_sets/dbapi.py| 73 ++-
5 files changed, 104 insertions(+), 4 deletions(-)
diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf
index c4ad2efca..2bf38e414 100644
--- a/cnf/sets/portage.conf
+++ b/cnf/sets/portage.conf
@@ -115,3 +115,8 @@ class = portage.sets.dbapi.ChangedDepsSet
class = portage.sets.dbapi.VariableSet
variable = INHERITED
includes = golang-base golang-build golang-vcs golang-vcs-snapshot go-module
+
+# Package set which contains all installed packages having one or more
+# unsatisfied runtime dependencies.
+[unsatisfied-deps]
+class = portage.sets.dbapi.UnsatisfiedDepsSet
diff --git a/doc/config/sets.docbook b/doc/config/sets.docbook
index eba98f468..015ec0c05 100644
--- a/doc/config/sets.docbook
+++ b/doc/config/sets.docbook
@@ -610,6 +610,13 @@
+
+ portage.sets.dbapi.UnsatisfiedDepsSet
+
+ Package set which contains all installed packages
+ having one or more unsatisfied runtime dependencies.
+
+
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 1946f49df..ba2592bba 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1000,11 +1000,18 @@ def _calc_depclean(settings, trees, ldpath_mtimes,
msg.append("%s" % (parent,))
msg.append("")
msg.extend(textwrap.wrap(
- "Have you forgotten to do a complete update
prior " + \
- "to depclean? The most comprehensive command
for this " + \
+ "Have you forgotten to resolve unsatisfied
dependencies prior "
+ "to depclean? The simplest possible command for
this "
"purpose is as follows:", 65
))
msg.append("")
+ msg.append(" " + \
+ good("emerge @unsatisfied-deps"))
+ msg.append("")
+ msg.extend(textwrap.wrap(
+ "The most comprehensive possible update command
is this:", 65
+ ))
+ msg.append("")
msg.append(" " + \
good("emerge --update --newuse --deep
--with-bdeps=y @world"))
msg.append("")
@@ -1018,8 +1025,14 @@ def _calc_depclean(settings, trees, ldpath_mtimes,
msg.extend(textwrap.wrap(
"Also, note that it may be necessary to
manually uninstall " + \
"packages that no longer exist in the
repository, since " + \
- "it may not be possible to satisfy their
dependencies.", 65
+ "it may not be possible to satisfy their
dependencies."
+
Re: [gentoo-portage-dev] [PATCH v2 2/2] ebuild.5: Add eapply documentation
On 4/10/21 6:23 PM, Nekun wrote: > Signed-off-by: Nekun > --- > man/ebuild.5 | 20 > 1 file changed, 20 insertions(+) Thank you! I've merged these and noted it on the bug here: https://bugs.gentoo.org/698244#c1 Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] emerge: use parse_intermixed_args when available (bug 784566)
The included unit test case previously failed with this error:
emerge: error: unrecognized arguments: dev-libs/A
Bug: https://bugs.gentoo.org/784566
Signed-off-by: Zac Medico
---
lib/_emerge/main.py | 2 +-
lib/portage/tests/emerge/test_simple.py | 10 +-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 31e690584..191be6479 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -834,7 +834,7 @@ def parse_opts(tmpcmdline, silent=False):
tmpcmdline = insert_optional_args(tmpcmdline)
- myoptions = parser.parse_args(args=tmpcmdline)
+ myoptions = getattr(parser, "parse_intermixed_args",
parser.parse_args)(args=tmpcmdline)
if myoptions.alert in true_y:
myoptions.alert = True
diff --git a/lib/portage/tests/emerge/test_simple.py
b/lib/portage/tests/emerge/test_simple.py
index 6e282337f..5b110407f 100644
--- a/lib/portage/tests/emerge/test_simple.py
+++ b/lib/portage/tests/emerge/test_simple.py
@@ -1,6 +1,7 @@
# Copyright 2011-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import argparse
import subprocess
import portage
@@ -289,7 +290,14 @@ call_has_and_best_version() {
port=binhost_server.server_port,
path=binhost_remote_path)
- test_commands = (
+ test_commands = ()
+
+ if hasattr(argparse.ArgumentParser, "parse_intermixed_args"):
+ test_commands += (
+ emerge_cmd + ("--oneshot", "dev-libs/A", "-v",
"dev-libs/A"),
+ )
+
+ test_commands += (
emerge_cmd + ("--usepkgonly", "--root", cross_root,
"--quickpkg-direct=y", "--quickpkg-direct-root", "/", "dev-libs/A"),
emerge_cmd + ("--usepkgonly", "--quickpkg-direct=y",
"--quickpkg-direct-root", cross_root, "dev-libs/A"),
env_update_cmd,
--
2.26.2
Re: [gentoo-portage-dev] profile-formats not respected ?
On 4/20/21 5:03 AM, Joakim Tjernlund wrote:
> On Mon, 2021-04-19 at 14:10 -0700, Zac Medico wrote:
>> On 4/19/21 6:36 AM, Joakim Tjernlund wrote:
>>> I got an embedded ppc32 system which I build in a QEMU user chroot and I
>>> cannot
>>> make the profile-formats = portage-2 profile-bashrcs profile-set in my own
>>> profiles layout.conf
>>> work for me.
>>> Seems like portage just ignores this setting and I cannot understand why.
>>> Any pointers?
>>>
>>> Â Jocke
>>>
>>
>> Hopefully this command will provide a clue:
>>
>> python -c 'import portage;
>> print(portage.settings._locations_manager.profiles_complex)'
>
> Got some progress. I got a profile.bashrc:
> .../my-overlay/profiles/cusfpv3/profile.bashrc
> where I setup PKG_INSTALL_MASK/INSTALL_MASK and this does not work in
> portage-3.0.18. I can echo the vars. and see them set but portage ignores
> these settings.
> This system was previously using a very old portage, 2.3.76 and that was fine.
>
>
> Another strange thing is my own set, i need to keep that at
> .../my-overlay/{sets, sets.conf}
> In our amd64 DE profile I can have my sets at
> .../my-overlay/profiles/infinera/sets , why is that?
A setting like this in /etc/portage/sets.conf would do it:
[my-overlay-profile-sets]
class = portage.sets.files.StaticFileSet
multiset = true
directory = /my-overlay/profiles/infinera/sets
world-candidate = true
> Anyhow, here is my profile printout:
> ./print-port.py
> (_profile_node(location='/usr/portage/profiles/base',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/default/linux',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',),
> eapi='5', allow_build_id=False, show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/arch/base',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/arch/powerpc',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/default/linux/powerpc',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5',
> allow_build_id=False, show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/arch/base',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/arch/powerpc',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/arch/powerpc/ppc32',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5',
> allow_build_id=False, show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/default/linux/powerpc/ppc32',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/releases',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/releases/17.0',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=False),
> _profile_node(location='/usr/portage/profiles/default/linux/powerpc/ppc32/17.0',
> portage1_directories=True, user_config=False,
> profile_formats=('portage-1-compat',), eapi='5', allow_build_id=False,
> show_deprecated_warning=True),
> _profile_node(location='/usr/local/portage/tmv3-target-
> overlay/profiles/cusfpv3', portage1_directories=True, user_config=False,
> profile_formats=('profile-bashrcs', 'portage-2', 'profile-set'), eapi='5',
> allow_build_id=False,
> show_deprecated_warning=True), _profile_node(location='/etc/portage/profile',
> portage1_directories=True, user_config=True,
> profile_formats=('profile-bashrcs', 'profile-set'),
> eapi=None, allow_build_id=True, show_deprecated_warning=False))
>
Are these effective profile_formats now consistent with your
expectations, and do they now behave as you would expect?
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] profile-formats not respected ?
On 4/19/21 6:36 AM, Joakim Tjernlund wrote: > I got an embedded ppc32 system which I build in a QEMU user chroot and I > cannot > make the profile-formats = portage-2 profile-bashrcs profile-set in my own > profiles layout.conf > work for me. > Seems like portage just ignores this setting and I cannot understand why. > Any pointers? > > Jocke > Hopefully this command will provide a clue: python -c 'import portage; print(portage.settings._locations_manager.profiles_complex)' -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] make.conf.5: Sugest PORTAGE_LOG_FILTER_FILE_CMD supervisor for cat fallback (bug 781854)
If PORTAGE_LOG_FILTER_FILE_CMD fails after exec, then output will be lost. Therefore, suggest to use bash as a supervisor, with fallback to cat. Bug: https://bugs.gentoo.org/781854 Signed-off-by: Zac Medico --- man/make.conf.5 | 15 +-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/man/make.conf.5 b/man/make.conf.5 index 8d551c95e..8a4a2ae30 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -1,4 +1,4 @@ -.TH "MAKE.CONF" "5" "Feb 2021" "Portage VERSION" "Portage" +.TH "MAKE.CONF" "5" "Apr 2021" "Portage VERSION" "Portage" .SH "NAME" make.conf \- custom settings for Portage .SH "SYNOPSIS" @@ -998,7 +998,18 @@ will set idle io priority. For more information about ionice, see .B PORTAGE_LOG_FILTER_FILE_CMD This variable specifies a command that filters build log output to a log file. In order to filter ANSI escape codes from build logs, -\fBansifilter\fR(1) is a convenient setting for this variable. +\fBansifilter\fR(1) is a convenient setting for this variable. Generally, +PORTAGE_LOG_FILTER_FILE_CMD should include a supervisor that falls back +to cat if the real filter command fails after exec. For example, a +supervisor is needed for ansifilter, in case it fails after exec due to +a problem resolving libstdc++ during a gcc upgrade. +.br +.I Example: +.nf +# Use bash as a supervisor, for fallback to cat if asifilter fails +# after exec due to a problem resolving libstdc++ during a gcc upgrade. +PORTAGE_LOG_FILTER_FILE_CMD="bash -c \\"ansifilter; exec cat\\"" +.fi .TP .B PORTAGE_LOGDIR This variable defines the directory in which per\-ebuild logs are kept. -- 2.26.2
Re: [gentoo-portage-dev] Implement new userpatch feature in existing eclass?
On 4/4/21 10:54 AM, Nekun wrote:
> Hi all.
>
> Recently, I start working on optional atom specifiers feature in
> userpatch facility: if package directory name starts with percent sign,
> following word threated as a regular Portage atom, e.g
> "/etc/portage/patches/sys-kernel/%<=gentoo-sources-5.4" ==
> "<=sys-kernel/gentoo-sources-5.4". This might be very useful in cases
> when patches applied to minor updates, but major update breaks it (e.g.,
> in Linux kernel), so I want to specify smth like "=gentoo-sources-5.4*".
> I added new command in portageq to match two atoms and call it from
> eapply_user function in phase-function.sh, in same manner as
> has_version/best_version are called it. But recently I found that
> eapply_user implemented in Portage only in EAPI 6, and there is its
> predecessor, epatch_user, implemented in epatch.eclass. So, ebuilds with
> EAPI<6 (I found 4463 in last gentoo snapshot) will ignore new "atomic"
> patch directories. Obviously, this is rather confusing, unacceptable
> behaviour.
>
> Can I patch epatch.eclass in gentoo repository to implement new
> userpatch facility for older EAPIs? I guess that EAPI version is
> considered as stable, unchangeable behaviour of all functions, but in
> other side, this feature doesn't changes anything existing: old
> userpatch semantics preserves and order of applying
> (${P}-${PR},${P},${PN}) not changed, seeking for atoms added at tail.
>
Today, I count only 2445, or 8.3% of ebuilds have EAPI 5.
I imagine that the migration is moving along, since we deprecated EAPI 5
on 2020-11-26 here:
https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=b2e281bb698eb93704e1987dc4df1cf2dd3c2cff
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] Implement new userpatch feature in existing eclass?
On 4/4/21 10:54 AM, Nekun wrote:
> Hi all.
>
> Recently, I start working on optional atom specifiers feature in
> userpatch facility: if package directory name starts with percent sign,
> following word threated as a regular Portage atom, e.g
> "/etc/portage/patches/sys-kernel/%<=gentoo-sources-5.4" ==
> "<=sys-kernel/gentoo-sources-5.4". This might be very useful in cases
> when patches applied to minor updates, but major update breaks it (e.g.,
> in Linux kernel), so I want to specify smth like "=gentoo-sources-5.4*".
> I added new command in portageq to match two atoms and call it from
> eapply_user function in phase-function.sh, in same manner as
> has_version/best_version are called it. But recently I found that
> eapply_user implemented in Portage only in EAPI 6, and there is its
> predecessor, epatch_user, implemented in epatch.eclass. So, ebuilds with
> EAPI<6 (I found 4463 in last gentoo snapshot) will ignore new "atomic"
> patch directories. Obviously, this is rather confusing, unacceptable
> behaviour.
>
> Can I patch epatch.eclass in gentoo repository to implement new
> userpatch facility for older EAPIs? I guess that EAPI version is
> considered as stable, unchangeable behaviour of all functions, but in
> other side, this feature doesn't changes anything existing: old
> userpatch semantics preserves and order of applying
> (${P}-${PR},${P},${PN}) not changed, seeking for atoms added at tail.
>
Today, I count only 2445, or 8.3% of ebuilds have EAPI 5.
I imagine that the migration is moving along, since we deprecated EAPI 5
on 2020-11-26 here:
https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=b2e281bb698eb93704e1987dc4df1cf2dd3c2cff
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Use atomic_ofstream as Context Manager i.e., with-statement contexts
On 3/8/21 11:25 PM, Florian Schmaus wrote: > With [1: e93e6d65fa1c] atomic_ofstream became a Context Manager. This > commit transforms three further call sites of atomic_ofstream() to use > with-statement contexts for easier readability and increased > robustness against resource leaks. > > 1: e93e6d65fa1ca75f676a227f7918f8b6d747425c >Make atomic_ofstream a Context Manager > > Signed-off-by: Florian Schmaus > --- > lib/_emerge/BlockerCache.py| 6 +++--- > lib/portage/dbapi/_VdbMetadataDelta.py | 11 +-- > lib/portage/dbapi/vartree.py | 6 +++--- Merged. Thank you! https://gitweb.gentoo.org/proj/portage.git/commit/?id=8879cef9006d2277453aaee407c234a2d1bc47ba -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Mark EAPIs "4-python" and "5-progress" as deprecated
On 3/4/21 11:35 AM, Matt Turner wrote: > Signed-off-by: Matt Turner > --- > I've asked Arfrever multiple times if these are still used anywhere, and > he seemingly has not responded intentionally. > > According to https://bugs.gentoo.org/174536#c27 these EAPIs were only > used in Arfrever's personal overlay, and even in 2012 there were > questions about why they were supported in portage. > > The "Progress Overlay" does contain ebuilds using these EAPIs but it has > not been updated since 2018 and doesn't look like it is useful at this > point. > > lib/portage/__init__.py | 8 > 1 file changed, 4 insertions(+), 4 deletions(-) > > diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py > index 24c9d8b89..184db6ae2 100644 > --- a/lib/portage/__init__.py > +++ b/lib/portage/__init__.py > @@ -465,16 +465,16 @@ def abssymlink(symlink, target=None): > _doebuild_manifest_exempt_depend = 0 > > _testing_eapis = frozenset([ > - "4-python", > - "5-progress", > ]) > _deprecated_eapis = frozenset([ > + "3_pre1", > + "3_pre2", > "4_pre1", > + "4-python", > "4-slot-abi", > - "3_pre2", > - "3_pre1", > "5_pre1", > "5_pre2", > + "5-progress", > "6_pre1", > "7_pre1", > ]) > Merged, thanks! https://gitweb.gentoo.org/proj/portage.git/commit/?id=67cf9c2b05042de37f36f5b6840c450128a065bd -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Use asyncio.subprocess.Process directly
On 3/4/21 11:24 AM, Matt Turner wrote:
> With no need to support Python 2, we can remove our private
> implementation.
>
> Signed-off-by: Matt Turner
> ---
> I don't know how to test this. I intentionally broke the return value of
> create_subprocess_exec and didn't see any bad results.
>
> lib/portage/util/futures/_asyncio/__init__.py | 8 +-
> lib/portage/util/futures/_asyncio/process.py | 116 --
> 2 files changed, 4 insertions(+), 120 deletions(-)
> delete mode 100644 lib/portage/util/futures/_asyncio/process.py
Merged, thanks!
https://gitweb.gentoo.org/proj/portage.git/commit/?id=1e843f853a9afe82d599e6ab09064147ddc1d271
> diff --git a/lib/portage/util/futures/_asyncio/__init__.py
> b/lib/portage/util/futures/_asyncio/__init__.py
> index 5590963f1..207e7205d 100644
> --- a/lib/portage/util/futures/_asyncio/__init__.py
> +++ b/lib/portage/util/futures/_asyncio/__init__.py
> @@ -25,6 +25,7 @@ import types
> import weakref
>
> import asyncio as _real_asyncio
> +from asyncio.subprocess import Process
>
> try:
> import threading
> @@ -138,7 +138,7 @@ def create_subprocess_exec(*args, **kwargs):
>
> result = loop.create_future()
>
> - result.set_result(_Process(subprocess.Popen(
> + result.set_result(Process(subprocess.Popen(
> args,
> stdin=kwargs.pop('stdin', None),
> stdout=kwargs.pop('stdout', None),
The above area is actually no longer used, since we should always have a
_AsyncioEventLoop instance here, and we can remove the EventLoop class now.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH 3/3] lib: Remove outdated Python 2 comments
On 3/4/21 11:23 AM, Matt Turner wrote:
> Fixes: 788c0e8bb ("Remove from __future__ import unicode_literals")
> Signed-off-by: Matt Turner
> ---
> bin/egencache | 2 --
> lib/_emerge/Package.py | 9 -
> lib/_emerge/Scheduler.py| 2 --
> lib/_emerge/UseFlagDisplay.py | 2 --
> lib/_emerge/resolver/output.py | 2 --
> lib/portage/cache/flat_hash.py | 3 ---
> lib/portage/tests/unicode/test_string_format.py | 9 -
> lib/portage/util/digraph.py | 3 ---
> 8 files changed, 32 deletions(-)
Series looks good. Merged. Thanks!
https://gitweb.gentoo.org/proj/portage.git/commit/?id=9003c5201c6503ddad9237bcffbc6f775567661b
https://gitweb.gentoo.org/proj/portage.git/commit/?id=af100c65ebf7fd84307a84819602a934ebb0741c
https://gitweb.gentoo.org/proj/portage.git/commit/?id=21c6c0c1088ded78397594bfd78102361f8d837b
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] emerge: make --binpkg-respect-use=y imply --autounmask-use=n
If --binpkg-respect-use=y is given explicitly, then it implies
--autounmask-use=n, because these options naturally oppose
eachother.
Bug: https://bugs.gentoo.org/773469
Signed-off-by: Zac Medico
---
lib/_emerge/create_depgraph_params.py | 22 ++---
lib/portage/tests/resolver/test_useflags.py | 20 +--
man/emerge.1| 11 +++
3 files changed, 32 insertions(+), 21 deletions(-)
diff --git a/lib/_emerge/create_depgraph_params.py
b/lib/_emerge/create_depgraph_params.py
index 25dd2a1b4..267600fb6 100644
--- a/lib/_emerge/create_depgraph_params.py
+++ b/lib/_emerge/create_depgraph_params.py
@@ -41,12 +41,22 @@ def create_depgraph_params(myopts, myaction):
# binpkg_changed_deps: reject binary packages with outdated deps
myparams = {"recurse" : True}
+ binpkg_respect_use = myopts.get("--binpkg-respect-use")
+ if binpkg_respect_use is not None:
+ myparams["binpkg_respect_use"] = binpkg_respect_use
+ elif "--usepkgonly" not in myopts:
+ # If --binpkg-respect-use is not explicitly specified, we enable
+ # the behavior automatically (like requested in bug #297549), as
+ # long as it doesn't strongly conflict with other options that
+ # have been specified.
+ myparams["binpkg_respect_use"] = "auto"
+
autounmask_keep_keywords = myopts.get("--autounmask-keep-keywords")
autounmask_keep_masks = myopts.get("--autounmask-keep-masks")
autounmask = myopts.get("--autounmask")
autounmask_license = myopts.get('--autounmask-license', 'y' if
autounmask is True else 'n')
- autounmask_use = myopts.get('--autounmask-use')
+ autounmask_use = 'n' if myparams.get('binpkg_respect_use') == 'y' else
myopts.get('--autounmask-use')
if autounmask == 'n':
autounmask = False
else:
@@ -153,16 +163,6 @@ def create_depgraph_params(myopts, myaction):
'--update' in myopts:
myparams['rebuilt_binaries'] = True
- binpkg_respect_use = myopts.get('--binpkg-respect-use')
- if binpkg_respect_use is not None:
- myparams['binpkg_respect_use'] = binpkg_respect_use
- elif '--usepkgonly' not in myopts:
- # If --binpkg-respect-use is not explicitly specified, we enable
- # the behavior automatically (like requested in bug #297549), as
- # long as it doesn't strongly conflict with other options that
- # have been specified.
- myparams['binpkg_respect_use'] = 'auto'
-
binpkg_changed_deps = myopts.get('--binpkg-changed-deps')
if binpkg_changed_deps is not None:
myparams['binpkg_changed_deps'] = binpkg_changed_deps
diff --git a/lib/portage/tests/resolver/test_useflags.py
b/lib/portage/tests/resolver/test_useflags.py
index d66da0866..b799e62ff 100644
--- a/lib/portage/tests/resolver/test_useflags.py
+++ b/lib/portage/tests/resolver/test_useflags.py
@@ -46,15 +46,23 @@ class UseFlagsTestCase(TestCase):
success = True,
mergelist = ["dev-libs/A-1"]),
- # In the unit test case for bug 773469, the
--autounmask-backtrack option
- # is needed in order to trigger the
--binpkg-respect-use=y behavior that
- # appears confusingly similar to --binpkg-respect-use=n
behavior.
+ # For bug 773469, we wanted --binpkg-respect-use=y to
trigger a
+ # slot collision. Instead, a combination of default
--autounmask-use
+ # combined with --autounmask-backtrack=y from
EMERGE_DEFAULT_OPTS
+ # triggered this behavior which appeared confusingly
similar to
+ #--binpkg-respect-use=n behavior.
+ #ResolverPlaygroundTestCase(
+ # ["dev-libs/C", "dev-libs/D"],
+ # options={"--usepkg": True,
"--binpkg-respect-use": "y", "--autounmask-backtrack": "y"},
+ # success=True,
+ # use_changes={"dev-libs/C-1": {"abi_x86_32":
True}},
+ # mergelist=["[binary]dev-libs/C-1",
"[binary]dev-libs/D-1"],
ResolverPlaygroundTestCase(
["dev-libs/C", "dev-libs/D"],
options={"--usepkg": True,
"--binpkg-respect-use": "y", "--autounmask-backtrack": "y"},
- success=True,
-
[gentoo-portage-dev] Planning to publish portage releases on pypi
I'd like to begin publishing portage releases on pypi here: https://pypi.org/project/portage It won't allow me to create the the project, so I've opened this ticket to claim it: https://github.com/pypa/pypi-support/issues/934 -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v3] emirrordist: add --content-db option required for content-hash layout (bug 756778)
Add a --content-db option which is required for the content-hash
layout because its file listings return content digests instead of
distfile names.
The content db serves to translate content digests to distfiles
names, and distfiles names to content digests. All keys have a
prefix separated by a colon. For digest keys, the prefix is the
hash algorithm name. For filename keys, the prefix is "filename".
The value associated with a digest key is a set of file names. The
value associated with a distfile key is a set of content revisions.
Each content revision is expressed as a dictionary of digests which
is suitable for construction of a DistfileName instance.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico
---
[PATCH v3] changed the value associated with a digest key is a set
of file name, and fixed ContentDB.remove to preserved independent
references to identical content (like removing one of multiple
hardlinks).
lib/portage/_emirrordist/Config.py | 8 +-
lib/portage/_emirrordist/ContentDB.py| 178 +++
lib/portage/_emirrordist/DeletionIterator.py | 25 ++-
lib/portage/_emirrordist/DeletionTask.py | 8 +
lib/portage/_emirrordist/FetchTask.py| 5 +-
lib/portage/_emirrordist/main.py | 15 +-
lib/portage/package/ebuild/fetch.py | 8 +-
lib/portage/tests/ebuild/test_fetch.py | 14 ++
man/emirrordist.1| 6 +-
9 files changed, 256 insertions(+), 11 deletions(-)
create mode 100644 lib/portage/_emirrordist/ContentDB.py
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index 4bee4f45e..cfe944040 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -10,6 +10,7 @@ import time
from portage import os
from portage.package.ebuild.fetch import MirrorLayoutConfig
from portage.util import grabdict, grablines
+from .ContentDB import ContentDB
class Config:
def __init__(self, options, portdb, event_loop):
@@ -65,6 +66,11 @@ class Config:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
+ self.content_db = None
+ if options.content_db is not None:
+ self.content_db = ContentDB(self._open_shelve(
+ options.content_db, 'content'))
+
self.deletion_db = None
if options.deletion_db is not None:
self.deletion_db = self._open_shelve(
diff --git a/lib/portage/_emirrordist/ContentDB.py
b/lib/portage/_emirrordist/ContentDB.py
new file mode 100644
index 0..7084cecff
--- /dev/null
+++ b/lib/portage/_emirrordist/ContentDB.py
@@ -0,0 +1,178 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import operator
+import shelve
+import typing
+
+from portage.package.ebuild.fetch import DistfileName
+
+
+class ContentDB:
+ """
+ The content db serves to translate content digests to distfiles
+ names, and distfiles names to content digests. All keys have a
+ prefix separated by a colon. For digest keys, the prefix is the
+ hash algorithm name. For filename keys, the prefix is "filename".
+
+ The value associated with a digest key is a set of file names. The
+ value associated with a distfile key is a set of content revisions.
+ Each content revision is expressed as a dictionary of digests which
+ is suitable for construction of a DistfileName instance.
+ """
+
+ def __init__(self, shelve_instance: shelve.Shelf):
+ self._shelve = shelve_instance
+
+ def add(self, filename: DistfileName):
+ """
+ Add file name and digests.
+
+ @param filename: file name with digests attribute
+ """
+ distfile_str = str(filename)
+ distfile_key = "filename:{}".format(distfile_str)
+ for k, v in filename.digests.items():
+ if k != "size":
+ digest_key = "{}:{}".format(k, v).lower()
+ try:
+ digest_files = self._shelve[digest_key]
+ except KeyError:
+ digest_files = set()
+ digest_files.add(distfile_str)
+ self._shelve[digest_key] = digest_files
+ try:
+ content_revisions = self._shelve[distfile_key]
+
[gentoo-portage-dev] [PATCH] make.globals: make FEATURES=-binpkg-multi-instance sticky for existing installs
Add a _compat_upgrade.binpkg_multi_instance script that the ebuild
can call in pkg_preinst in order to maintain a backward-compatible
FEATURES=-binpkg-multi-instance default on existing installs where
enabling binpkg-multi-instance could cause disruption.
Bug: https://bugs.gentoo.org/772785
Signed-off-by: Zac Medico
---
.../_compat_upgrade/binpkg_multi_instance.py | 33 +++
1 file changed, 33 insertions(+)
create mode 100644 lib/portage/_compat_upgrade/binpkg_multi_instance.py
diff --git a/lib/portage/_compat_upgrade/binpkg_multi_instance.py
b/lib/portage/_compat_upgrade/binpkg_multi_instance.py
new file mode 100644
index 0..bfeb72baa
--- /dev/null
+++ b/lib/portage/_compat_upgrade/binpkg_multi_instance.py
@@ -0,0 +1,33 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import portage
+from portage import os
+from portage.const import GLOBAL_CONFIG_PATH
+
+COMPAT_FEATURES = 'FEATURES="${FEATURES} -binpkg-multi-instance"'
+
+
+def main():
+ """
+ If the current installation is still has binpkg-multi-instance
+ disabled, then patch make.globals inside ${ED} to maintain backward
+ compatibility. This is intended to be called from the ebuild as
+ follows:
+
+ pkg_preinst() {
+ python_setup
+ env -u FEATURES
+
PYTHONPATH="${D%/}$(python_get_sitedir)${PYTHONPATH:+:${PYTHONPATH}}" \
+ "${PYTHON}" -m
portage._compat_upgrade.binpkg_multi_instance || die
+ }
+ """
+ if 'binpkg-multi-instance' not in portage.settings.features:
+ portage.output.EOutput().einfo('Setting make.globals default {}
for backward compatibility'.format(COMPAT_FEATURES))
+ config_path = os.path.join(os.environ['ED'],
GLOBAL_CONFIG_PATH.lstrip(os.sep), 'make.globals')
+ with open(config_path, 'at') as f:
+ f.write("{}\n".format(COMPAT_FEATURES))
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
[gentoo-portage-dev] [PATCH v2] emirrordist: add --content-db option required for content-hash layout (bug 756778)
Add a --content-db option which is required for the content-hash
layout because its file listings return content digests instead of
distfile names.
The content db serves to translate content digests to distfiles
names, and distfiles names to content digests. All keys have a
prefix separated by a colon. For digest keys, the prefix is the
hash algorithm name. For filename keys, the prefix is "filename".
The value associated with a digest key is a plain filename. The
value associated with a distfile key is a set of content revisions.
Each content revision is expressed as a dictionary of digests which
is suitable for construction of a DistfileName instance.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico
---
[PATCH v2] Split out ContentDB class and associate distfile key
with a set of content revisions, where each content revision is
expressed as a dictionary of digests.
lib/portage/_emirrordist/Config.py | 8 +-
lib/portage/_emirrordist/ContentDB.py| 158 +++
lib/portage/_emirrordist/DeletionIterator.py | 25 ++-
lib/portage/_emirrordist/DeletionTask.py | 8 +
lib/portage/_emirrordist/FetchTask.py| 5 +-
lib/portage/_emirrordist/main.py | 15 +-
lib/portage/tests/ebuild/test_fetch.py | 14 ++
man/emirrordist.1| 6 +-
8 files changed, 232 insertions(+), 7 deletions(-)
create mode 100644 lib/portage/_emirrordist/ContentDB.py
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index 4bee4f45e..cfe944040 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -10,6 +10,7 @@ import time
from portage import os
from portage.package.ebuild.fetch import MirrorLayoutConfig
from portage.util import grabdict, grablines
+from .ContentDB import ContentDB
class Config:
def __init__(self, options, portdb, event_loop):
@@ -65,6 +66,11 @@ class Config:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
+ self.content_db = None
+ if options.content_db is not None:
+ self.content_db = ContentDB(self._open_shelve(
+ options.content_db, 'content'))
+
self.deletion_db = None
if options.deletion_db is not None:
self.deletion_db = self._open_shelve(
diff --git a/lib/portage/_emirrordist/ContentDB.py
b/lib/portage/_emirrordist/ContentDB.py
new file mode 100644
index 0..60e6ef39d
--- /dev/null
+++ b/lib/portage/_emirrordist/ContentDB.py
@@ -0,0 +1,158 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import operator
+import shelve
+import typing
+
+from portage.package.ebuild.fetch import DistfileName
+
+
+class ContentDB:
+ """
+ The content db serves to translate content digests to distfiles
+ names, and distfiles names to content digests. All keys have a
+ prefix separated by a colon. For digest keys, the prefix is the
+ hash algorithm name. For filename keys, the prefix is "filename".
+
+ The value associated with a digest key is a plain filename. The
+ value associated with a distfile key is a set of content revisions.
+ Each content revision is expressed as a dictionary of digests which
+ is suitable for construction of a DistfileName instance.
+ """
+
+ def __init__(self, shelve_instance: shelve.Shelf):
+ self._shelve = shelve_instance
+
+ def add(self, filename: DistfileName):
+ """
+ Add file name and digests.
+
+ @param filename: file name with digests attribute
+ """
+ distfile_str = str(filename)
+ distfile_key = "filename:{}".format(distfile_str)
+ for k, v in filename.digests.items():
+ if k != "size":
+ self._shelve["{}:{}".format(k, v).lower()] =
distfile_str
+ try:
+ content_revisions = self._shelve[distfile_key]
+ except KeyError:
+ content_revisions = set()
+
+ revision_key = tuple(
+ sorted(
+ (
+ (algo.lower(),
filename.digests[algo].lower())
+ for algo in filename.digests
+
[gentoo-portage-dev] [PATCH] emirrordist: add --content-db option required for content-hash layout (bug 756778)
Add a --content-db option which is required for the content-hash
layout because its file listings return content digests instead of
distfile names.
The content db includes a reverse mapping, for use during garbage
collection. All keys have a prefix separated by a colon. For digest
keys, the prefix is the hash algorithm name. For filename keys,
the prefix is "filename". The values for digest keys are plain
filenames, and the values for distfile keys are dictionaries
of digests suitable for construction of DistfileName instances.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico
---
lib/portage/_emirrordist/Config.py | 7 +++-
lib/portage/_emirrordist/DeletionIterator.py | 38 ++--
lib/portage/_emirrordist/DeletionTask.py | 26 ++
lib/portage/_emirrordist/FetchTask.py| 16 -
lib/portage/_emirrordist/main.py | 15 +++-
lib/portage/tests/ebuild/test_fetch.py | 14
man/emirrordist.1| 6 +++-
7 files changed, 116 insertions(+), 6 deletions(-)
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index 4bee4f45e..53f6582fe 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -65,6 +65,11 @@ class Config:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
+ self.content_db = None
+ if options.content_db is not None:
+ self.content_db = self._open_shelve(
+ options.content_db, 'content')
+
self.deletion_db = None
if options.deletion_db is not None:
self.deletion_db = self._open_shelve(
diff --git a/lib/portage/_emirrordist/DeletionIterator.py
b/lib/portage/_emirrordist/DeletionIterator.py
index 08985ed6c..24fb096bf 100644
--- a/lib/portage/_emirrordist/DeletionIterator.py
+++ b/lib/portage/_emirrordist/DeletionIterator.py
@@ -1,10 +1,12 @@
-# Copyright 2013-2019 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import logging
import stat
+import typing
from portage import os
+from portage.package.ebuild.fetch import DistfileName
from .DeletionTask import DeletionTask
class DeletionIterator:
@@ -12,6 +14,37 @@ class DeletionIterator:
def __init__(self, config):
self._config = config
+ def _map_filename(self, filename: typing.Union[str, DistfileName]) ->
typing.Union[str, DistfileName]:
+ """
+ Map a filename listed by the layout get_filenames method,
+ translating it from a content digest to a distfile name.
+ If filename is already a distfile name, then it will pass
+ through unchanged.
+
+ @param filename: A filename listed by layout get_filenames
+ @return: The distfile name, mapped from the corresponding
+ content digest when necessary
+ """
+ if not isinstance(filename, DistfileName):
+ if self._config.content_db is not None:
+ distfile_key = "filename:{}".format(filename)
+ try:
+ digests =
self._config.content_db[distfile_key]
+ except KeyError:
+ pass
+ else:
+ return DistfileName(filename,
digests=digests)
+ return DistfileName(filename)
+ if filename.digests and self._config.content_db is not None:
+ for k, v in filename.digests.items():
+ digest_key = "{}:{}".format(k, v).lower()
+ try:
+ distfile_str =
self._config.content_db[digest_key]
+ except KeyError:
+ continue
+ return DistfileName(distfile_str, digests={k:v})
+ return filename
+
def __iter__(self):
distdir = self._config.options.distfiles
file_owners = self._config.file_owners
@@ -22,7 +55,8 @@ class DeletionIterator:
start_time = self._config.start_time
distfiles_set = set()
for layout in self._config.layouts:
- distfiles_set.update(layout.get_filenames(distdir))
+
[gentoo-portage-dev] Re: [PATCH] repoman: revert preserve_old_lib deprecation (bug 480244)
On 2/23/21 3:41 PM, Zac Medico wrote: > Repoman should not report that preserve_old_lib is deprecated, > since preserve-libs is not covered by PMS. > This reverts commit 49cbc17bf7b99be586e158c1bd588cfe91dfe58c. > > Bug: https://bugs.gentoo.org/480244 > Bug: https://bugs.gentoo.org/692486 > Signed-off-by: Zac Medico > --- > repoman/cnf/linechecks/linechecks.yaml| 2 +- > .../lib/repoman/modules/linechecks/deprecated/deprecated.py | 4 ++-- > 2 files changed, 3 insertions(+), 3 deletions(-) Due to controversy, I would like to withdraw this patch, and call upon interested parties to help decide an appropriate course of action or inaction for https://bugs.gentoo.org/480244. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] repoman: revert preserve_old_lib deprecation (bug 480244)
Repoman should not report that preserve_old_lib is deprecated, since preserve-libs is not covered by PMS. This reverts commit 49cbc17bf7b99be586e158c1bd588cfe91dfe58c. Bug: https://bugs.gentoo.org/480244 Bug: https://bugs.gentoo.org/692486 Signed-off-by: Zac Medico --- repoman/cnf/linechecks/linechecks.yaml| 2 +- .../lib/repoman/modules/linechecks/deprecated/deprecated.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/repoman/cnf/linechecks/linechecks.yaml b/repoman/cnf/linechecks/linechecks.yaml index 2182b467a..7c040e2b2 100644 --- a/repoman/cnf/linechecks/linechecks.yaml +++ b/repoman/cnf/linechecks/linechecks.yaml @@ -25,7 +25,7 @@ errors: DEPRECATED_BINDNOW_FLAGS: 'Deprecated bindnow-flags call' EAPI_DEFINED_AFTER_INHERIT: 'EAPI defined after inherit' NO_AS_NEEDED: 'Upstream asneeded linking bug (no-as-needed)' -PRESERVE_OLD_LIB: 'Ebuild calls deprecated preserve_old_lib' +PRESERVE_OLD_LIB: 'Upstream ABI change workaround on line: %d' BUILT_WITH_USE: 'built_with_use' NO_OFFSET_WITH_HELPERS: 'Helper function is used with D, ROOT, ED, EROOT or EPREFIX' USEQ_ERROR: 'Ebuild calls deprecated useq function' diff --git a/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py b/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py index d1a590f1d..ad488c53a 100644 --- a/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py +++ b/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py @@ -19,8 +19,8 @@ class DeprecatedHasq(LineCheck): class PreserveOldLib(LineCheck): - """Check for calls to the deprecated preserve_old_lib function.""" - repoman_check_name = 'ebuild.minorsyn' + """Check for calls to the preserve_old_lib function.""" + repoman_check_name = 'upstream.workaround' re = re.compile(r'.*preserve_old_lib') error = 'PRESERVE_OLD_LIB' -- 2.26.2
[gentoo-portage-dev] [PATCH] Add content-hash distfiles layout (bug 756778)
From: Daniel Robbins
The content-hash layout is identical to the filename-hash layout,
except for these two differences:
1) A content digest is used instead of a filename digest.
2) The final element of the path returned from the get_path method
corresponds to the complete content digest. The path is a
function of the content digest alone.
Motivations to use the content-hash layout instead of the
filename-hash layout may include:
1) Since the file path is independent of the file name, file
name collisions cannot occur. This makes the content-hash
layout suitable for storage of multiple types of files (not
only gentoo distfiles). For example, it can be used to store
distfiles for multiple linux distros within the same tree,
with automatic deduplication based on content digest. This
layout can be used to store and distribute practically anything
(including binary packages for example).
2) Allows multiple revisions for the same distfiles name. An
existing distfile can be updated, and if a user still has an
older copy of an ebuild repository (or an overlay), then a user
can successfully fetch a desired revision of the distfile as
long as it has not been purged from the mirror.
3) File integrity data is integrated into the layout itself,
making it very simple to verify the integrity of any file that
it contains. The only tool required is an implementation of
the chosen hash algorithm.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico
---
lib/portage/package/ebuild/fetch.py| 160 +++--
lib/portage/tests/ebuild/test_fetch.py | 40 ++-
2 files changed, 184 insertions(+), 16 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py
b/lib/portage/package/ebuild/fetch.py
index e0fecaf23..7d2ef93bf 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -344,6 +344,31 @@ _size_suffix_map = {
}
+class DistfileName(str):
+ def __new__(cls, s, digests=None):
+ return str.__new__(cls, s)
+
+ def __init__(self, s, digests=None):
+ super().__init__()
+ self.digests = {} if digests is None else digests
+
+ def digests_equal(self, other):
+ """
+ Test if digests compare equal to those of another instance.
+ """
+ if not isinstance(other, DistfileName):
+ return False
+ matches = []
+ for algo, digest in self.digests.items():
+ other_digest = other.digests.get(algo)
+ if other_digest is not None:
+ if other_digest == digest:
+ matches.append(algo)
+ else:
+ return False
+ return bool(matches)
+
+
class FlatLayout:
def get_path(self, filename):
return filename
@@ -413,6 +438,90 @@ class FilenameHashLayout:
return False
+class ContentHashLayout(FilenameHashLayout):
+ """
+ The content-hash layout is identical to the filename-hash layout,
+ except for these two differences:
+
+ 1) A content digest is used instead of a filename digest.
+ 2) The final element of the path returned from the get_path method
+ corresponds to the complete content digest. The path is a
+ function of the content digest alone.
+
+ Motivations to use the content-hash layout instead of the
+ filename-hash layout may include:
+
+ 1) Since the file path is independent of the file name, file
+ name collisions cannot occur. This makes the content-hash
+ layout suitable for storage of multiple types of files (not
+ only gentoo distfiles). For example, it can be used to store
+ distfiles for multiple linux distros within the same tree,
+ with automatic deduplication based on content digest. This
+ layout can be used to store and distribute practically anything
+ (including binary packages for example).
+
+ 2) Allows multiple revisions for the same distfiles name. An
+ existing distfile can be updated, and if a user still has an
+ older copy of an ebuild repository (or an overlay), then a user
+ can successfully fetch a desired revision of the distfile as
+ long as it has not been purged from the mirror.
+
+ 3) File integrity data is integrated into the layout itself,
+ making it very simple to verify the integrity of any file that
+ it contains. The only tool required is an implementation of
+ the chosen hash algorithm.
+ """
+
+ def get_path(self, filename):
+ """
+ For content-hash, the path is a function of the content digest
alone.
+ The final element of the path re
[gentoo-portage-dev] [PATCH] make.defaults: prevent USE="${USE} ..." misbehavior
Discard parent profile USE from the expand_map variable before
it is used to evaluate a child profile. This prevents accidents
triggered by USE="${USE} ..." settngs at the top of make.defaults
which caused parent profile USE to override parent profile
package.use settings.
Bug: https://bugs.gentoo.org/771549
Signed-off-by: Zac Medico
---
lib/portage/package/ebuild/config.py | 13 ++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/lib/portage/package/ebuild/config.py
b/lib/portage/package/ebuild/config.py
index e5ec681af..638c72959 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -612,9 +612,16 @@ class config:
mygcfg = {}
if profiles_complex:
- mygcfg_dlists =
[getconfig(os.path.join(x.location, "make.defaults"),
- tolerant=tolerant, expand=expand_map,
recursive=x.portage1_directories)
- for x in profiles_complex]
+ mygcfg_dlists = []
+ for x in profiles_complex:
+ # Prevent accidents triggered by
USE="${USE} ..." settings
+ # at the top of make.defaults which
caused parent profile
+ # USE to override parent profile
package.use settings.
+ expand_map.pop("USE", None)
+ mygcfg_dlists.append(
+
getconfig(os.path.join(x.location, "make.defaults"),
+ tolerant=tolerant,
expand=expand_map,
+
recursive=x.portage1_directories))
self._make_defaults = mygcfg_dlists
mygcfg = stack_dicts(mygcfg_dlists,
incrementals=self.incrementals)
--
2.26.2
[gentoo-portage-dev] [PATCH] portage.getpid: call os.getpid() lazily
Call os.getpid() lazily, which eliminates getpid calls when possible after os.fork() in the portage.process module. Bug: https://bugs.gentoo.org/767913 Signed-off-by: Zac Medico --- lib/portage/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py index 3c9f78497..24c9d8b89 100644 --- a/lib/portage/__init__.py +++ b/lib/portage/__init__.py @@ -375,7 +375,7 @@ _sync_mode = False class _ForkWatcher: @staticmethod def hook(_ForkWatcher): - _ForkWatcher.current_pid = _os.getpid() + _ForkWatcher.current_pid = None # Force instantiation of a new event loop policy as a workaround # for https://bugs.python.org/issue22087. asyncio.set_event_loop_policy(None) @@ -388,6 +388,8 @@ def getpid(): """ Cached version of os.getpid(). ForkProcess updates the cache. """ + if _ForkWatcher.current_pid is None: + _ForkWatcher.current_pid = _os.getpid() return _ForkWatcher.current_pid def _get_stdin(): -- 2.26.2
[gentoo-portage-dev] [PATCH] emerge: disable --autounmask-license by default
Disable --autounmask-license by default, in order to limit user
exposure to risks associated with package.license changes.
The changes that this option suggests are only intended to be
accepted when a user has made a conscious decision to accept
the corresponding license(s). Creation of package.license
changes introduces a risk that users may erroneously accept the
changes due to some kind of accident or misunderstanding,
rather than due to conscious decisions about licenses.
These risks provide motivation to disable --autounmask-license
by default. The --autounmask-use option will remain as the
only autounmask option that is still enabled by default.
The unit tests demonstrate interactions between --autounmask
and --autounmask-license options. The --autounmask option
enables --autounmask-license unless --autounmask-license=n
has been specified. If --autounmask=n is used to disable
autounmask, then --autounmask-license=y has no effect.
Bug: https://bugs.gentoo.org/766773
Signed-off-by: Zac Medico
---
lib/_emerge/create_depgraph_params.py | 8 +++---
lib/portage/tests/resolver/test_autounmask.py | 25 +--
man/emerge.1 | 11 +++-
3 files changed, 31 insertions(+), 13 deletions(-)
diff --git a/lib/_emerge/create_depgraph_params.py
b/lib/_emerge/create_depgraph_params.py
index 0d0e07b9c..25dd2a1b4 100644
--- a/lib/_emerge/create_depgraph_params.py
+++ b/lib/_emerge/create_depgraph_params.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import logging
@@ -45,7 +45,7 @@ def create_depgraph_params(myopts, myaction):
autounmask_keep_masks = myopts.get("--autounmask-keep-masks")
autounmask = myopts.get("--autounmask")
- autounmask_license = myopts.get('--autounmask-license')
+ autounmask_license = myopts.get('--autounmask-license', 'y' if
autounmask is True else 'n')
autounmask_use = myopts.get('--autounmask-use')
if autounmask == 'n':
autounmask = False
@@ -53,7 +53,7 @@ def create_depgraph_params(myopts, myaction):
if autounmask is None:
if autounmask_use in (None, 'y'):
autounmask = True
- elif autounmask_license in (None, 'y'):
+ if autounmask_license in ('y',):
autounmask = True
# Do not enable package.accept_keywords or package.mask
@@ -67,7 +67,7 @@ def create_depgraph_params(myopts, myaction):
myparams['autounmask'] = autounmask
myparams['autounmask_keep_use'] = True if autounmask_use == 'n' else
False
- myparams['autounmask_keep_license'] = True if autounmask_license == 'n'
else False
+ myparams['autounmask_keep_license'] = False if autounmask_license ==
'y' else True
myparams['autounmask_keep_keywords'] = False if
autounmask_keep_keywords in (None, 'n') else True
myparams['autounmask_keep_masks'] = False if autounmask_keep_masks in
(None, 'n') else True
diff --git a/lib/portage/tests/resolver/test_autounmask.py
b/lib/portage/tests/resolver/test_autounmask.py
index a3bf0ff94..86ae4bbf6 100644
--- a/lib/portage/tests/resolver/test_autounmask.py
+++ b/lib/portage/tests/resolver/test_autounmask.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2019 Gentoo Authors
+# Copyright 2010-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -440,13 +440,34 @@ class AutounmaskTestCase(TestCase):
mergelist=["dev-libs/A-1"],
license_changes={ "dev-libs/A-1":
set(["TEST"]) }),
- # Test default --autounmask-license
+ # Test that --autounmask enables
--autounmask-license
ResolverPlaygroundTestCase(
["=dev-libs/A-1"],
+ options={"--autounmask": True},
success=False,
mergelist=["dev-libs/A-1"],
license_changes={ "dev-libs/A-1":
set(["TEST"]) }),
+ # Test that --autounmask-license is not enabled
by default
+ ResolverPlaygroundTestCase(
+ ["=dev-libs/A-1"],
+ success=False,
+ ),
+
+ # Test that --autounmask does not override
--autounmask-license=n
+ ResolverPlaygroundTestCase(
+
[gentoo-portage-dev] [PATCH] emaint --fix merges: add -y, --yes option
Since the emaint --fix merges uses emerge --ask, add
a -y, --yes option to use --ask=n instead.
Bug: https://bugs.gentoo.org/766767
Signed-off-by: Zac Medico
---
lib/portage/emaint/modules/merges/__init__.py | 14 --
lib/portage/emaint/modules/merges/merges.py | 11 +++
man/emaint.1 | 6 +-
3 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/lib/portage/emaint/modules/merges/__init__.py
b/lib/portage/emaint/modules/merges/__init__.py
index 89aa758a0..449f39dce 100644
--- a/lib/portage/emaint/modules/merges/__init__.py
+++ b/lib/portage/emaint/modules/merges/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2014 Gentoo Foundation
+# Copyright 2005-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
doc = """Scan for failed merges and fix them."""
@@ -26,7 +26,17 @@ module_spec = {
'action': 'store_true',
'func': 'purge'
}
- }
+ },
+ 'opt_desc': {
+ 'yes': {
+ "short": "-y",
+ "long": "--yes",
+ "help": ("(merges submodule only): Do
not prompt for "
+ "emerge invocations"),
+ "action": "store_true",
+ "dest": "yes",
+ }
+ },
}
}
}
diff --git a/lib/portage/emaint/modules/merges/merges.py
b/lib/portage/emaint/modules/merges/merges.py
index 775dc59d2..d60916f1e 100644
--- a/lib/portage/emaint/modules/merges/merges.py
+++ b/lib/portage/emaint/modules/merges/merges.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2020 Gentoo Authors
+# Copyright 2005-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import portage
@@ -186,7 +186,7 @@ class MergesHandler:
pkg_atoms.add(pkg_atom)
- def _emerge_pkg_atoms(self, module_output, pkg_atoms):
+ def _emerge_pkg_atoms(self, module_output, pkg_atoms, yes=False):
"""
Emerge the specified packages atoms.
@@ -194,6 +194,8 @@ class MergesHandler:
@type module_output: Class
@param pkg_atoms: packages atoms to emerge
@type pkg_atoms: set
+ @param yes: do not prompt for emerge invocations
+ @type yes: bool
@rtype: list
@return: List of results
"""
@@ -206,7 +208,7 @@ class MergesHandler:
portage._python_interpreter,
'-b',
os.path.join(EPREFIX or '/', 'usr', 'bin', 'emerge'),
- '--ask',
+ '--ask=n' if yes else '--ask',
'--quiet',
'--oneshot',
'--complete-graph=y'
@@ -265,7 +267,8 @@ class MergesHandler:
errors.append(', '.join(sorted(failed_pkgs)))
return (False, errors)
self._remove_failed_dirs(failed_pkgs)
- results = self._emerge_pkg_atoms(module_output, pkg_atoms)
+ results = self._emerge_pkg_atoms(module_output, pkg_atoms,
+ yes=kwargs.get('options', {}).get("yes", False))
# list any new failed merges
for pkg in sorted(self._scan()):
results.append("'%s' still found as a failed merge." %
pkg)
diff --git a/man/emaint.1 b/man/emaint.1
index d244756e9..c9f8ab939 100644
--- a/man/emaint.1
+++ b/man/emaint.1
@@ -1,4 +1,4 @@
-.TH "EMAINT" "1" "Jan 2017" "Portage VERSION" "Portage"
+.TH "EMAINT" "1" "Jan 2021" "Portage VERSION" "Portage"
.SH NAME
emaint \- performs package management related system health checks and
maintenance
.SH SYNOPSIS
@@ -82,6 +82,10 @@ OPTION
.BR \-t \ \fINUM\fR,\ \-\-time \ \fINUM\fR
Changes the minimum age \fINUM\fR (in days) of the logs to be listed or
deleted.
+.SH OPTIONS merges command only
+.TP
+.BR \-y ", " \-\-yes
+Do not prompt for emerge invocations.
.SH OPTIONS sync command only
.TP
.BR \-a ", " \-\-auto
--
2.26.2
[gentoo-portage-dev] [PATCH] binarytree.move_ent: copy on write for package move
Copy on write when applying package moves, and silently
skip package moves when the same move has already been
applied to the same build of the package. Since the old
package instance is preserved, it avoids the problem
of having enries for deleted packages remain in the
package index. We can simply assume that the package
will be deleted by eclean-pkg when its time comes.
Bug: https://bugs.gentoo.org/766012
Signed-off-by: Zac Medico
---
lib/portage/dbapi/bintree.py | 40 ---
lib/portage/emaint/modules/move/move.py | 13 ++--
lib/portage/tests/update/test_move_ent.py | 7 ++--
3 files changed, 44 insertions(+), 16 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 180e48c3b..76fca5523 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -31,6 +31,7 @@ from portage.exception import AlarmSignal,
InvalidPackageName, \
ParseError, PortageException
from portage.localization import _
from portage.package.ebuild.profile_iuse import iter_iuse_vars
+from portage.util.file_copy import copyfile
from portage.util.futures import asyncio
from portage.util.futures.compat_coroutine import coroutine
from portage.util.futures.executor.fork import ForkExecutor
@@ -483,6 +484,17 @@ class binarytree:
myoldpkg = catsplit(mycpv)[1]
mynewpkg = catsplit(mynewcpv)[1]
+ # If this update has already been applied to the same
+ # package build then silently continue.
+ applied = False
+ for maybe_applied in
self.dbapi.match('={}'.format(mynewcpv)):
+ if maybe_applied.build_time == mycpv.build_time:
+ applied = True
+ break
+
+ if applied:
+ continue
+
if (mynewpkg != myoldpkg) and
self.dbapi.cpv_exists(mynewcpv):
writemsg(_("!!! Cannot update binary:
Destination exists.\n"),
noiselevel=-1)
@@ -513,24 +525,30 @@ class binarytree:
mydata[_unicode_encode(mynewpkg +
'.ebuild',
encoding=_encodings['repo.content'])] = ebuild_data
- mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
-
- self.dbapi.cpv_remove(mycpv)
- del self._pkg_paths[self.dbapi._instance_key(mycpv)]
metadata = self.dbapi._aux_cache_slot_dict()
for k in self.dbapi._aux_cache_keys:
v = mydata.get(_unicode_encode(k))
if v is not None:
v = _unicode_decode(v)
metadata[k] = " ".join(v.split())
+
+ # Create a copy of the old version of the package and
+ # apply the update to it. Leave behind the old version,
+ # assuming that it will be deleted by eclean-pkg when
its
+ # time comes.
mynewcpv = _pkg_str(mynewcpv, metadata=metadata,
db=self.dbapi)
- new_path = self.getname(mynewcpv)
- self._pkg_paths[
- self.dbapi._instance_key(mynewcpv)] =
new_path[len(self.pkgdir)+1:]
- if new_path != tbz2path:
- self._ensure_dir(os.path.dirname(new_path))
- _movefile(tbz2path, new_path,
mysettings=self.settings)
- self.inject(mynewcpv)
+ update_path = self.getname(mynewcpv, allocate_new=True)
+ ".partial"
+ self._ensure_dir(os.path.dirname(update_path))
+ update_path_lock = None
+ try:
+ update_path_lock = lockfile(update_path,
wantnewlockfile=True)
+ copyfile(tbz2path, update_path)
+ mytbz2 = portage.xpak.tbz2(update_path)
+
mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
+ self.inject(mynewcpv, filename=update_path)
+ finally:
+ if update_path_lock is not None:
+ unlockfile(update_path_lock)
return moves
diff --git a/lib/portage/emaint/modules/move/move.py
b/lib/portage/emaint/modules/move/move.py
index 8fc3269ca..2a95e99c4 100644
--- a/lib/portage/emaint/modules/move/move.py
+++ b/lib/portage/emaint/modules/move/move.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2020 Gentoo
Re: [gentoo-portage-dev] [PATCH] Add @changed-subslot package set
On 1/18/21 8:42 PM, Alec Warner wrote: > On Mon, Jan 18, 2021 at 8:09 PM Zac Medico wrote: >> >> On 1/18/21 6:07 PM, Alec Warner wrote: >>> On Fri, Jan 15, 2021 at 6:47 PM Matt Turner wrote: >>>> >>>> This set is the upgradable packages for which the highest visible >>>> version has a different subslot than the currently installed version. >>>> >>>> The primary purpose of this feature is for use in catalyst builds. We >>>> update the "seed" stage3 before using it to build a new stage1. >>>> >>>> Updating the entire stage is expensive and unnecessary (since we're >>>> going to build the latest packages in stage1 and then rebuild everything >>>> in stage3). >>>> >>>> What we definitely do need to update in the original stage3 however, is >>>> any package that would trigger a subslot rebuild. >>>> >>>> For example: gcc links with libmpfr.so from dev-libs/mpfr. mpfr's SONAME >>>> changes from libmpfr.so.4 (SLOT="0/4") to libmpfr.so.6 (SLOT="0/6"). If >>>> the seed stage's dev-libs/mpfr is not updated before emerging gcc, gcc >>>> will link with libmpfr.so.4, but the latest version of dev-libs/mpfr >>>> will be built and libmpfr.so.6 included into the stage1. Since the old >>>> libmpfr.so.4 is not included in the stage1, gcc will not work, breaking >>>> subsequent stage builds. >>>> >>>> Our current options to update the seed are too large a hammer (e.g., >>>> "--update --deep --newuse @world" or "--update --deep --newuse >>>> --complete-graph --rebuild-if-new-ver gcc") and spend too much time >>>> updating seed stages for no gain beyond updating only packages for whom >>>> the subslot has changed. >>>> >>>> With this set, catalyst will likely use >>>> >>>> emerge @changed-subslot --ignore-built-slot-operator-deps y >>>> >>>> to update the seed stage. >>>> >>>> Thank you to Zac Medico for showing me how to do this. >>>> >>>> Bug: https://bugs.gentoo.org/739004 >>>> Signed-off-by: Matt Turner >>>> --- >>>> cnf/sets/portage.conf | 5 + >>>> lib/portage/_sets/dbapi.py | 39 +- >>>> 2 files changed, 43 insertions(+), 1 deletion(-) >>>> >>>> diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf >>>> index 22f0fa3a5..5651a9c53 100644 >>>> --- a/cnf/sets/portage.conf >>>> +++ b/cnf/sets/portage.conf >>>> @@ -84,6 +84,11 @@ exclude-files = /usr/bin/Xorg >>>> [rebuilt-binaries] >>>> class = portage.sets.dbapi.RebuiltBinaries >>>> >>>> +# Installed packages for which the subslot of the highest visible ebuild >>>> +# version is different than the currently installed version. >>>> +[changed-subslot] >>>> +class = portage.sets.dbapi.SubslotChangedSet >>>> + >>>> # Installed packages for which the highest visible ebuild >>>> # version is lower than the currently installed version. >>>> [downgrade] >>>> diff --git a/lib/portage/_sets/dbapi.py b/lib/portage/_sets/dbapi.py >>>> index 52367c4a6..46ba5c17d 100644 >>>> --- a/lib/portage/_sets/dbapi.py >>>> +++ b/lib/portage/_sets/dbapi.py >>>> @@ -15,7 +15,7 @@ from portage._sets import SetConfigError, get_boolean >>>> import portage >>>> >>>> __all__ = ["CategorySet", "ChangedDepsSet", "DowngradeSet", >>>> - "EverythingSet", "OwnerSet", "VariableSet"] >>>> + "EverythingSet", "OwnerSet", "SubslotChangedSet", "VariableSet"] >>>> >>>> class EverythingSet(PackageSet): >>>> _operations = ["merge"] >>>> @@ -167,6 +167,43 @@ class VariableSet(EverythingSet): >>>> >>>> singleBuilder = classmethod(singleBuilder) >>>> >>>> +class SubslotChangedSet(PackageSet): >>>> + >>>> + _operations = ["merge", "unmerge"] >>>> + >>>> + description = "Package set which contains all packages " + \ >>>> + "for which the subslot of the highest visible ebuild is " >>>> + \ >&g
Re: [gentoo-portage-dev] [PATCH] Add @changed-subslot package set
On 1/18/21 6:07 PM, Alec Warner wrote:
> On Fri, Jan 15, 2021 at 6:47 PM Matt Turner wrote:
>>
>> This set is the upgradable packages for which the highest visible
>> version has a different subslot than the currently installed version.
>>
>> The primary purpose of this feature is for use in catalyst builds. We
>> update the "seed" stage3 before using it to build a new stage1.
>>
>> Updating the entire stage is expensive and unnecessary (since we're
>> going to build the latest packages in stage1 and then rebuild everything
>> in stage3).
>>
>> What we definitely do need to update in the original stage3 however, is
>> any package that would trigger a subslot rebuild.
>>
>> For example: gcc links with libmpfr.so from dev-libs/mpfr. mpfr's SONAME
>> changes from libmpfr.so.4 (SLOT="0/4") to libmpfr.so.6 (SLOT="0/6"). If
>> the seed stage's dev-libs/mpfr is not updated before emerging gcc, gcc
>> will link with libmpfr.so.4, but the latest version of dev-libs/mpfr
>> will be built and libmpfr.so.6 included into the stage1. Since the old
>> libmpfr.so.4 is not included in the stage1, gcc will not work, breaking
>> subsequent stage builds.
>>
>> Our current options to update the seed are too large a hammer (e.g.,
>> "--update --deep --newuse @world" or "--update --deep --newuse
>> --complete-graph --rebuild-if-new-ver gcc") and spend too much time
>> updating seed stages for no gain beyond updating only packages for whom
>> the subslot has changed.
>>
>> With this set, catalyst will likely use
>>
>> emerge @changed-subslot --ignore-built-slot-operator-deps y
>>
>> to update the seed stage.
>>
>> Thank you to Zac Medico for showing me how to do this.
>>
>> Bug: https://bugs.gentoo.org/739004
>> Signed-off-by: Matt Turner
>> ---
>> cnf/sets/portage.conf | 5 +
>> lib/portage/_sets/dbapi.py | 39 +-
>> 2 files changed, 43 insertions(+), 1 deletion(-)
>>
>> diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf
>> index 22f0fa3a5..5651a9c53 100644
>> --- a/cnf/sets/portage.conf
>> +++ b/cnf/sets/portage.conf
>> @@ -84,6 +84,11 @@ exclude-files = /usr/bin/Xorg
>> [rebuilt-binaries]
>> class = portage.sets.dbapi.RebuiltBinaries
>>
>> +# Installed packages for which the subslot of the highest visible ebuild
>> +# version is different than the currently installed version.
>> +[changed-subslot]
>> +class = portage.sets.dbapi.SubslotChangedSet
>> +
>> # Installed packages for which the highest visible ebuild
>> # version is lower than the currently installed version.
>> [downgrade]
>> diff --git a/lib/portage/_sets/dbapi.py b/lib/portage/_sets/dbapi.py
>> index 52367c4a6..46ba5c17d 100644
>> --- a/lib/portage/_sets/dbapi.py
>> +++ b/lib/portage/_sets/dbapi.py
>> @@ -15,7 +15,7 @@ from portage._sets import SetConfigError, get_boolean
>> import portage
>>
>> __all__ = ["CategorySet", "ChangedDepsSet", "DowngradeSet",
>> - "EverythingSet", "OwnerSet", "VariableSet"]
>> + "EverythingSet", "OwnerSet", "SubslotChangedSet", "VariableSet"]
>>
>> class EverythingSet(PackageSet):
>> _operations = ["merge"]
>> @@ -167,6 +167,43 @@ class VariableSet(EverythingSet):
>>
>> singleBuilder = classmethod(singleBuilder)
>>
>> +class SubslotChangedSet(PackageSet):
>> +
>> + _operations = ["merge", "unmerge"]
>> +
>> + description = "Package set which contains all packages " + \
>> + "for which the subslot of the highest visible ebuild is " + \
>> + "different than the currently installed version."
>
> description = ("string1",
>"string2",
>"string3")
>
> vs concat + \ for line continuation?
>
> -A
We also got flak on irc about the classmethod(singleBuilder) usage as
opposed to @classmethod. This package set is formatted exactly like
others in the file, so it's just a copy / paste thing.
On the topic of python formatting, maybe we should use something like
https://github.com/psf/black to automate it?
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH gentoolkit] equery: Remove 'changes' subcommand
On 1/4/21 10:44 AM, Matt Turner wrote: > ChangeLogs have been gone from gentoo.git since the beginning, and > Council agreed in 2016 to allow Infra to decide whether to distribute > them through rsync, which they have decided not to do [1]. > > [1] https://projects.gentoo.org/council/meeting-logs/20160410-summary.txt > > Signed-off-by: Matt Turner > --- > pym/gentoolkit/equery/__init__.py | 1 - > pym/gentoolkit/equery/changes.py| 184 > pym/gentoolkit/helpers.py | 173 -- > pym/gentoolkit/test/equery/test_init.py | 1 - > pym/gentoolkit/test/test_helpers.py | 47 -- > 5 files changed, 406 deletions(-) > delete mode 100644 pym/gentoolkit/equery/changes.py Looks good. We should also be able to remove the egencache --update-changelogs option. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] Re: [PATCH] global_event_loop: return running loop for current thread
On 1/4/21 1:06 AM, Zac Medico wrote: > Like asyncio.get_event_loop(), return the running loop for the > current thread if there is one, and otherwise construct a new > one if needed. This allows the _safe_loop function to become > synonymous with the global_event_loop function. > > Bug: https://bugs.gentoo.org/763339 > Signed-off-by: Zac Medico > --- > .../util/_eventloop/global_event_loop.py | 28 ++- > lib/portage/util/futures/_asyncio/__init__.py | 22 ++- > 2 files changed, 17 insertions(+), 33 deletions(-) For the case of "loop running in non-main thread" of API consumer, this change makes portage compatible with PEP 492 coroutines with async and await syntax. Portage internals can safely begin using async / await syntax instead of compat_coroutine. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] global_event_loop: return running loop for current thread
Like asyncio.get_event_loop(), return the running loop for the
current thread if there is one, and otherwise construct a new
one if needed. This allows the _safe_loop function to become
synonymous with the global_event_loop function.
Bug: https://bugs.gentoo.org/763339
Signed-off-by: Zac Medico
---
.../util/_eventloop/global_event_loop.py | 28 ++-
lib/portage/util/futures/_asyncio/__init__.py | 22 ++-
2 files changed, 17 insertions(+), 33 deletions(-)
diff --git a/lib/portage/util/_eventloop/global_event_loop.py
b/lib/portage/util/_eventloop/global_event_loop.py
index 413011178..bf314dc34 100644
--- a/lib/portage/util/_eventloop/global_event_loop.py
+++ b/lib/portage/util/_eventloop/global_event_loop.py
@@ -1,28 +1,4 @@
-# Copyright 2012-2020 Gentoo Authors
+# Copyright 2012-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import portage
-from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop
-
-_instances = {}
-
-
-def global_event_loop():
- """
- Get a global EventLoop (or compatible object) instance which
- belongs exclusively to the current process.
- """
-
- pid = portage.getpid()
- instance = _instances.get(pid)
- if instance is not None:
- return instance
-
- constructor = AsyncioEventLoop
-
- # Use the _asyncio_wrapper attribute, so that unit tests can compare
- # the reference to one retured from _wrap_loop(), since they should
- # not close the loop if it refers to a global event loop.
- instance = constructor()._asyncio_wrapper
- _instances[pid] = instance
- return instance
+from portage.util.futures._asyncio import _safe_loop as global_event_loop
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index d39f31786..ab1468d43 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018-2020 Gentoo Authors
+# Copyright 2018-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = (
@@ -37,9 +37,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures:compat_coroutine@_compat_coroutine',
)
from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as
_AsyncioEventLoop
-from portage.util._eventloop.global_event_loop import (
- global_event_loop as _global_event_loop,
-)
# pylint: disable=redefined-builtin
from portage.util.futures.futures import (
CancelledError,
@@ -238,7 +235,7 @@ def _wrap_loop(loop=None):
# The default loop returned by _wrap_loop should be consistent
# with global_event_loop, in order to avoid accidental registration
# of callbacks with a loop that is not intended to run.
- loop = loop or _global_event_loop()
+ loop = loop or _safe_loop()
return (loop if hasattr(loop, '_asyncio_wrapper')
else _AsyncioEventLoop(loop=loop))
@@ -267,8 +264,9 @@ def _safe_loop():
@rtype: asyncio.AbstractEventLoop (or compatible)
@return: event loop instance
"""
- if portage._internal_caller or threading.current_thread() is
threading.main_thread():
- return _global_event_loop()
+ loop = _get_running_loop()
+ if loop is not None:
+ return loop
thread_key = threading.get_ident()
with _thread_weakrefs.lock:
@@ -286,6 +284,16 @@ def _safe_loop():
return loop
+def _get_running_loop():
+ with _thread_weakrefs.lock:
+ if _thread_weakrefs.pid == portage.getpid():
+ try:
+ loop =
_thread_weakrefs.loops[threading.get_ident()]
+ except KeyError:
+ return None
+ return loop if loop.is_running() else None
+
+
def _thread_weakrefs_atexit():
with _thread_weakrefs.lock:
if _thread_weakrefs.pid == portage.getpid():
--
2.26.2
Re: [gentoo-portage-dev] [PATCH gentoolkit] eclean: Add --changed-iuse flag
On 1/2/21 4:08 PM, Matt Turner wrote: > Allows binpkgs to be deleted if they are not usable due to IUSE changes. > --- > Just kind of spitballing. I'm not sure about what USE flags we should > ignore or whether it should be configurable, etc. On one hand, deleting > binpkgs that don't have a newly added PYTHON_TARGET option might make > sense if your binhost is configured to rebuild the package. On the > other, you probably don't want to throw out amd64 binpkgs because > abi_riscv_* was added. The special case for abi_* flags is ugly. Why not do emerge emerge --changed-use, and ignore changed IUSE for flags that aren't enabled? -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH gentoolkit] bin: Add merge-driver-ekeyword
On 12/31/20 11:47 AM, Matt Turner wrote: > Since the KEYWORDS=... assignment is a single line, git struggles to > handle conflicts. When rebasing a series of commits that modify the > KEYWORDS=... it's usually easier to throw them away and reapply on the > new tree than it is to manually handle conflicts during the rebase. > > git allows a 'merge driver' program to handle conflicts; this program > handles conflicts in the KEYWORDS=... assignment. E.g., given an ebuild > with these keywords: > > KEYWORDS="~alpha amd64 arm arm64 ~hppa ppc ppc64 x86" > > One developer drops the ~alpha keyword and pushes to gentoo.git, and > another developer stabilizes hppa. Without this merge driver, git > requires the second developer to manually resolve the conflict which is > tedious and prone to mistakes when rebasing a long series of patches. > With the custom merge driver, it automatically resolves the conflict. > > To use the merge driver, configure your gentoo.git as such: > > gentoo.git/.git/config: > > [merge "keywords"] > name = KEYWORDS merge driver > driver = merge-driver-ekeyword %O %A %B %P > > gentoo.git/.git/info/attributes: > > *.ebuild merge=keywords > > Signed-off-by: Matt Turner > --- > v3: Address Zac's feedback: use tempfile.TemporaryDirectory Looks great! -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH gentoolkit] bin: Add merge-driver-ekeyword
On 12/28/20 5:09 PM, Zac Medico wrote:
> On 12/28/20 3:15 PM, Matt Turner wrote:
>> +def apply_keyword_changes(ebuild: str, pathname: str,
>> + changes: List[Tuple[Optional[str],
>> + Optional[str]]]) -> int:
>> +result: int = 0
>> +
>> +# ekeyword will only modify files named *.ebuild, so make a symlink
>> +ebuild_symlink: str = os.path.basename(pathname)
>> +os.symlink(ebuild, ebuild_symlink)
>
> Are we sure that the current working directory is an entirely safe place
> to create this symlink? A simple fix would be to use
> tempfile.TemporaryDirectory to create a temporary directory to hold the
> symlink. Or, we could change ekeyword to assume that an argument is an
> ebuild if os.path.isfile(arg) succeeds.
>
>> +for removals, additions in changes:
>> +args = []
>> +for rem in removals:
>> +# Drop leading '~' and '-' characters and prepend '^'
>> +i = 1 if rem[0] in ('~', '-') else 0
>> +args.append('^' + rem[i:])
>> +if additions:
>> +args.extend(additions)
>> +args.append(ebuild_symlink)
>> +
>> +result = ekeyword.main(args)
Another option is to bypass the ekeyword.main function, like this:
try:
ekeyword.process_ebuild(pathname, list(map(ekeyword.arg_to_op, args))
except Exception:
result = 1
traceback.print_exc()
else:
result = 0
>> +if result != 0:
>> +break
>> +
>> +os.remove(ebuild_symlink)
>> +return result
>
>
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v2] ekeyword: remove .ebuild file suffix requirement (bug 762331)
We'd like to use ekeyword in a git merge driver implementation, but the
files that the driver will pass to ekeyword do not necessarily have a
.ebuild suffix. Therefore, it would be handy to be able to distinguish
ebuild arguments some other way. If the ignorable_arg(arg) function
returns True and os.path.isfile(arg) returns True, then simply assume
that the argument is an ebuild.
Bug: https://bugs.gentoo.org/762331
Signed-off-by: Zac Medico
---
[PATCH v2] fix to respect the ignorable_arg function
pym/gentoolkit/ekeyword/ekeyword.py | 9 ++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/pym/gentoolkit/ekeyword/ekeyword.py
b/pym/gentoolkit/ekeyword/ekeyword.py
index 4e57c09..eeceed4 100755
--- a/pym/gentoolkit/ekeyword/ekeyword.py
+++ b/pym/gentoolkit/ekeyword/ekeyword.py
@@ -244,7 +244,7 @@ def process_content(ebuild, data, ops, arch_status=None,
verbose=0,
pass
else:
# Chop the full path and the .ebuild suffix.
- disp_name = os.path.basename(ebuild)[:-7]
+ disp_name, _, _ = os.path.basename(ebuild).partition('.ebuild')
def logit(msg):
print('%s: %s' % (disp_name, msg))
@@ -395,7 +395,9 @@ def args_to_work(args, arch_status=None, _repo=None,
quiet=0):
last_todo_arches = []
for arg in args:
- if arg.endswith('.ebuild'):
+ if ignorable_arg(arg, quiet=quiet):
+ pass
+ elif os.path.isfile(arg):
if not todo_arches:
todo_arches = last_todo_arches
work.append([arg, todo_arches])
@@ -405,7 +407,7 @@ def args_to_work(args, arch_status=None, _repo=None,
quiet=0):
op = arg_to_op(arg)
if not arch_status or op.arch in arch_status:
todo_arches.append(op)
- elif not ignorable_arg(arg, quiet=quiet):
+ else:
raise ValueError('unknown arch/argument: %s' %
arg)
if todo_arches:
@@ -475,6 +477,7 @@ def main(argv):
opts.style = 'color-inline'
arch_status = load_profile_data()
+ print(arch_status)
try:
work = args_to_work(work_args, arch_status=arch_status,
quiet=opts.quiet)
except ValueError as e:
--
2.26.2
[gentoo-portage-dev] [PATCH] ekeyword: remove .ebuild file suffix requirement (bug 762331)
We'd like to use ekeyword in a git merge driver implementation, but the
files that the driver will pass to ekeyword do not necessarily have a
.ebuild suffix. Therefore, it would be handy to be able to distinguish
ebuild arguments some other way.
Bug: https://bugs.gentoo.org/762331
Signed-off-by: Zac Medico
---
pym/gentoolkit/ekeyword/ekeyword.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/gentoolkit/ekeyword/ekeyword.py
b/pym/gentoolkit/ekeyword/ekeyword.py
index 4e57c09..665eee5 100755
--- a/pym/gentoolkit/ekeyword/ekeyword.py
+++ b/pym/gentoolkit/ekeyword/ekeyword.py
@@ -244,7 +244,7 @@ def process_content(ebuild, data, ops, arch_status=None,
verbose=0,
pass
else:
# Chop the full path and the .ebuild suffix.
- disp_name = os.path.basename(ebuild)[:-7]
+ disp_name, _, _ = os.path.basename(ebuild).partition('.ebuild')
def logit(msg):
print('%s: %s' % (disp_name, msg))
@@ -395,7 +395,7 @@ def args_to_work(args, arch_status=None, _repo=None,
quiet=0):
last_todo_arches = []
for arg in args:
- if arg.endswith('.ebuild'):
+ if os.path.isfile(arg):
if not todo_arches:
todo_arches = last_todo_arches
work.append([arg, todo_arches])
--
2.26.2
Re: [gentoo-portage-dev] [PATCH gentoolkit] bin: Add merge-driver-ekeyword
On 12/28/20 3:15 PM, Matt Turner wrote:
> +def apply_keyword_changes(ebuild: str, pathname: str,
> + changes: List[Tuple[Optional[str],
> + Optional[str]]]) -> int:
> +result: int = 0
> +
> +# ekeyword will only modify files named *.ebuild, so make a symlink
> +ebuild_symlink: str = os.path.basename(pathname)
> +os.symlink(ebuild, ebuild_symlink)
Are we sure that the current working directory is an entirely safe place
to create this symlink? A simple fix would be to use
tempfile.TemporaryDirectory to create a temporary directory to hold the
symlink. Or, we could change ekeyword to assume that an argument is an
ebuild if os.path.isfile(arg) succeeds.
> +for removals, additions in changes:
> +args = []
> +for rem in removals:
> +# Drop leading '~' and '-' characters and prepend '^'
> +i = 1 if rem[0] in ('~', '-') else 0
> +args.append('^' + rem[i:])
> +if additions:
> +args.extend(additions)
> +args.append(ebuild_symlink)
> +
> +result = ekeyword.main(args)
> +if result != 0:
> +break
> +
> +os.remove(ebuild_symlink)
> +return result
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Drop Python 2 comatibility in extension modules
On 12/24/20 10:14 AM, Mike Gilbert wrote: > Signed-off-by: Mike Gilbert > --- > src/portage_util_file_copy_reflink_linux.c | 10 +- > src/portage_util_libc.c| 10 +- > 2 files changed, 2 insertions(+), 18 deletions(-) Looks good. Please merge. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Adjust mangling of "arch" value from scanelf output
On 12/23/20 7:34 PM, Mike Gilbert wrote:
> scanelf may generate output that looks like this:
>
> ```
> UNKNOWN_TYPE;lib/firmware/ath10k/WCN3990/hw1.0/wlanmdsp.mbn;; - ;
> EM_ARM;lib/firmware/mediatek/mt8183/scp.img;; - ;
> ...
> ```
>
> Previously, we removed the first 3 characters of the first field and
> stored this as the "arch" in NEEDED.ELF.2. This unintentionally
> changes "UNKNOWN_TYPE" to "NOWN_TYPE".
>
> Instead, let's just remove the string "EM_" from the front.
>
> Signed-off-by: Mike Gilbert
> ---
> bin/misc-functions.sh | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
> index c2a16cbe0..d7009d7eb 100755
> --- a/bin/misc-functions.sh
> +++ b/bin/misc-functions.sh
> @@ -194,7 +194,7 @@ install_qa_check() {
> fi
>
> echo "${obj} ${needed}" >>
> "${PORTAGE_BUILDDIR}"/build-info/NEEDED
> - echo "${arch:3};${obj};${soname};${rpath};${needed}" >>
> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2
> + echo "${arch#EM_};${obj};${soname};${rpath};${needed}"
> >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2
> done }
>
> [ -n "${QA_SONAME_NO_SYMLINK}" ] && \
>
Look good. This won't cause any problems for portage since these files
are outside of the known multilib categories.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH gentoolkit 1/4] Remove imports from __future__
On 12/20/20 2:10 PM, Matt Turner wrote: > gentoolkit supports only Python 3.6+ now, so these are not used. > > Signed-off-by: Matt Turner The whole series LGTM (including "Remove unused sys imports" patch 5/4). -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v4] Use default asyncio event loop implementation in API consumer threads
Make the _safe_loop function return an AsyncioEventLoop instance, so that the default asyncio event loop implementation will be used in API consumer threads. This is possible because the underlying asyncio.get_event_loop() function returns a separate event loop for each thread. The AsyncioEventLoop _run_until_complete method will now appropriately handle a ValueError from signal.set_wakeup_fd(-1) if it is not called in the main thread. For external API consumers calling from a non-main thread, an asyncio loop must be registered for the current thread, or else an error will be raised like this: RuntimeError: There is no current event loop in thread 'Thread-1'. In order to avoid this RuntimeError, the external API consumer is responsible for setting an event loop and managing its lifecycle. This code will set an event loop for the current thread: asyncio.set_event_loop(asyncio.new_event_loop()) In order to avoid a ResourceWarning, the caller should also close the corresponding loop before the current thread terminates. Bug: https://bugs.gentoo.org/758755 Signed-off-by: Zac Medico --- [PATCH v4] treat external API consumers the same as interal callers if they call from the main thread, and document asyncio loop lifecycle management now required for external API consumers calling from a non-main thread .../util/_eventloop/asyncio_event_loop.py | 6 - lib/portage/util/futures/_asyncio/__init__.py | 26 ++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index 836f1c30a..4d7047ae8 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -121,4 +121,8 @@ class AsyncioEventLoop(_AbstractEventLoop): try: return self._loop.run_until_complete(future) finally: - self._wakeup_fd = signal.set_wakeup_fd(-1) + try: + self._wakeup_fd = signal.set_wakeup_fd(-1) + except ValueError: + # This is intended to fail when not called in the main thread. + pass diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index a902ad895..0b35c6daf 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -34,7 +34,6 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.futures.unix_events:_PortageEventLoopPolicy', 'portage.util.futures:compat_coroutine@_compat_coroutine', - 'portage.util._eventloop.EventLoop:EventLoop@_EventLoop', ) from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as _AsyncioEventLoop from portage.util._eventloop.global_event_loop import ( @@ -246,14 +245,27 @@ def _wrap_loop(loop=None): def _safe_loop(): """ Return an event loop that's safe to use within the current context. - For portage internal callers, this returns a globally shared event - loop instance. For external API consumers, this constructs a - temporary event loop instance that's safe to use in a non-main - thread (it does not override the global SIGCHLD handler). + For portage internal callers or external API consumers calling from + the main thread, this returns a globally shared event loop instance. + + For external API consumers calling from a non-main thread, an + asyncio loop must be registered for the current thread, or else an + error will be raised like this: + + RuntimeError: There is no current event loop in thread 'Thread-1'. + + In order to avoid this RuntimeError, the external API consumer + is responsible for setting an event loop and managing its lifecycle. + This code will set an event loop for the current thread: + + asyncio.set_event_loop(asyncio.new_event_loop()) + + In order to avoid a ResourceWarning, the caller should also close the + corresponding loop before the current thread terminates. @rtype: asyncio.AbstractEventLoop (or compatible) @return: event loop instance """ - if portage._internal_caller: + if portage._internal_caller or threading.current_thread() is threading.main_thread(): return _global_event_loop() - return _EventLoop(main=False) + return _AsyncioEventLoop() -- 2.26.2
[gentoo-portage-dev] Re: [PATCH v3] Use default asyncio event loop implementation in API consumer threads
Accidentally encrypted the last email. Here's an unencrypted version.
On 12/6/20 2:14 PM, Zac Medico wrote:
> Make the _safe_loop function return an AsyncioEventLoop instance,
> so that the default asyncio event loop implementation will be used
> in API consumer threads. This is possible because the underlying
> asyncio.get_event_loop() function returns a new event loop for
> each thread. The AsyncioEventLoop _run_until_complete method will
> now appropriately handle a ValueError from signal.set_wakeup_fd(-1)
> if it is not called in the main thread.
>
> Bug: https://bugs.gentoo.org/758755
> Signed-off-by: Zac Medico
> ---
> [PATCH v3] fixed AsyncioEventLoop _run_until_complete method to
> handle ValueError from signal.set_wakeup_fd(-1)
>
> lib/portage/util/_eventloop/asyncio_event_loop.py | 6 +-
> lib/portage/util/futures/_asyncio/__init__.py | 3 +--
> 2 files changed, 6 insertions(+), 3 deletions(-)
>
> diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py
> b/lib/portage/util/_eventloop/asyncio_event_loop.py
> index 836f1c30a..4d7047ae8 100644
> --- a/lib/portage/util/_eventloop/asyncio_event_loop.py
> +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
> @@ -121,4 +121,8 @@ class AsyncioEventLoop(_AbstractEventLoop):
> try:
> return self._loop.run_until_complete(future)
> finally:
> - self._wakeup_fd = signal.set_wakeup_fd(-1)
> + try:
> + self._wakeup_fd = signal.set_wakeup_fd(-1)
> + except ValueError:
> + # This is intended to fail when not called in
> the main thread.
> + pass
> diff --git a/lib/portage/util/futures/_asyncio/__init__.py
> b/lib/portage/util/futures/_asyncio/__init__.py
> index a902ad895..12013be00 100644
> --- a/lib/portage/util/futures/_asyncio/__init__.py
> +++ b/lib/portage/util/futures/_asyncio/__init__.py
> @@ -34,7 +34,6 @@ import portage
> portage.proxy.lazyimport.lazyimport(globals(),
> 'portage.util.futures.unix_events:_PortageEventLoopPolicy',
> 'portage.util.futures:compat_coroutine@_compat_coroutine',
> - 'portage.util._eventloop.EventLoop:EventLoop@_EventLoop',
> )
> from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as
> _AsyncioEventLoop
> from portage.util._eventloop.global_event_loop import (
> @@ -256,4 +255,4 @@ def _safe_loop():
> """
> if portage._internal_caller:
> return _global_event_loop()
> - return _EventLoop(main=False)
> + return _AsyncioEventLoop()
>
This fails if an event loop has not been created for the current thread:
File "/usr/lib/python3.8/asyncio/events.py", line 639, in get_event_loop
raise RuntimeError('There is no current event loop in thread %r.'
RuntimeError: There is no current event loop in thread 'Thread-1'.
However, if we automatically instantiate a loop for the current thread
then we will be responsible for closing it as well, or else we'll
eventually see a ResourceWarning like this:
/usr/lib/python3.8/asyncio/base_events.py:654: ResourceWarning: unclosed
event loop <_UnixSelectorEventLoop running=False closed=False debug=False>
_warn(f"unclosed event loop {self!r}", ResourceWarning, source=self)
ResourceWarning: Enable tracemalloc to get the object allocation traceback
So, I think it's probably best if we force the API consumer to manage
the lifecycle of an asyncio loop for each thread that it uses to call
the portage API.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-portage-dev] Re: [PATCH v3] Use default asyncio event loop implementation in API consumer threads
On 12/6/20 2:14 PM, Zac Medico wrote:
> Make the _safe_loop function return an AsyncioEventLoop instance,
> so that the default asyncio event loop implementation will be used
> in API consumer threads. This is possible because the underlying
> asyncio.get_event_loop() function returns a new event loop for
> each thread. The AsyncioEventLoop _run_until_complete method will
> now appropriately handle a ValueError from signal.set_wakeup_fd(-1)
> if it is not called in the main thread.
>
> Bug: https://bugs.gentoo.org/758755
> Signed-off-by: Zac Medico
> ---
> [PATCH v3] fixed AsyncioEventLoop _run_until_complete method to
> handle ValueError from signal.set_wakeup_fd(-1)
>
> lib/portage/util/_eventloop/asyncio_event_loop.py | 6 +-
> lib/portage/util/futures/_asyncio/__init__.py | 3 +--
> 2 files changed, 6 insertions(+), 3 deletions(-)
>
> diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py
> b/lib/portage/util/_eventloop/asyncio_event_loop.py
> index 836f1c30a..4d7047ae8 100644
> --- a/lib/portage/util/_eventloop/asyncio_event_loop.py
> +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
> @@ -121,4 +121,8 @@ class AsyncioEventLoop(_AbstractEventLoop):
> try:
> return self._loop.run_until_complete(future)
> finally:
> - self._wakeup_fd = signal.set_wakeup_fd(-1)
> + try:
> + self._wakeup_fd = signal.set_wakeup_fd(-1)
> + except ValueError:
> + # This is intended to fail when not called in
> the main thread.
> + pass
> diff --git a/lib/portage/util/futures/_asyncio/__init__.py
> b/lib/portage/util/futures/_asyncio/__init__.py
> index a902ad895..12013be00 100644
> --- a/lib/portage/util/futures/_asyncio/__init__.py
> +++ b/lib/portage/util/futures/_asyncio/__init__.py
> @@ -34,7 +34,6 @@ import portage
> portage.proxy.lazyimport.lazyimport(globals(),
> 'portage.util.futures.unix_events:_PortageEventLoopPolicy',
> 'portage.util.futures:compat_coroutine@_compat_coroutine',
> - 'portage.util._eventloop.EventLoop:EventLoop@_EventLoop',
> )
> from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as
> _AsyncioEventLoop
> from portage.util._eventloop.global_event_loop import (
> @@ -256,4 +255,4 @@ def _safe_loop():
> """
> if portage._internal_caller:
> return _global_event_loop()
> - return _EventLoop(main=False)
> + return _AsyncioEventLoop()
>
This fails if an event loop has not been created for the current thread:
File "/usr/lib/python3.8/asyncio/events.py", line 639, in get_event_loop
raise RuntimeError('There is no current event loop in thread %r.'
RuntimeError: There is no current event loop in thread 'Thread-1'.
However, if we automatically instantiate a loop for the current thread
then we will be responsible for closing it as well, or else we'll
eventually see a ResourceWarning like this:
/usr/lib/python3.8/asyncio/base_events.py:654: ResourceWarning: unclosed
event loop <_UnixSelectorEventLoop running=False closed=False debug=False>
_warn(f"unclosed event loop {self!r}", ResourceWarning, source=self)
ResourceWarning: Enable tracemalloc to get the object allocation traceback
So, I think it's probably best if we force the API consumer to manage
the lifecycle of an asyncio loop for each thread that it uses to call
the portage API.
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v3] Use default asyncio event loop implementation in API consumer threads
Make the _safe_loop function return an AsyncioEventLoop instance, so that the default asyncio event loop implementation will be used in API consumer threads. This is possible because the underlying asyncio.get_event_loop() function returns a new event loop for each thread. The AsyncioEventLoop _run_until_complete method will now appropriately handle a ValueError from signal.set_wakeup_fd(-1) if it is not called in the main thread. Bug: https://bugs.gentoo.org/758755 Signed-off-by: Zac Medico --- [PATCH v3] fixed AsyncioEventLoop _run_until_complete method to handle ValueError from signal.set_wakeup_fd(-1) lib/portage/util/_eventloop/asyncio_event_loop.py | 6 +- lib/portage/util/futures/_asyncio/__init__.py | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index 836f1c30a..4d7047ae8 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -121,4 +121,8 @@ class AsyncioEventLoop(_AbstractEventLoop): try: return self._loop.run_until_complete(future) finally: - self._wakeup_fd = signal.set_wakeup_fd(-1) + try: + self._wakeup_fd = signal.set_wakeup_fd(-1) + except ValueError: + # This is intended to fail when not called in the main thread. + pass diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index a902ad895..12013be00 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -34,7 +34,6 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.futures.unix_events:_PortageEventLoopPolicy', 'portage.util.futures:compat_coroutine@_compat_coroutine', - 'portage.util._eventloop.EventLoop:EventLoop@_EventLoop', ) from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as _AsyncioEventLoop from portage.util._eventloop.global_event_loop import ( @@ -256,4 +255,4 @@ def _safe_loop(): """ if portage._internal_caller: return _global_event_loop() - return _EventLoop(main=False) + return _AsyncioEventLoop() -- 2.26.2
[gentoo-portage-dev] [PATCH v2] Use default asyncio event loop implementation in API consumer threads
Make the _safe_loop function return an AsyncioEventLoop instance, so that the default asyncio event loop implementation will be used in API consumer threads. This is possible because the underlying asyncio.get_event_loop() function returns a new event loop for each thread. Bug: https://bugs.gentoo.org/758755 Signed-off-by: Zac Medico --- [PATCH v2] fixed _safe_loop function to return a new AsyncioEventLoop per thread lib/portage/util/futures/_asyncio/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index a902ad895..12013be00 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -34,7 +34,6 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.futures.unix_events:_PortageEventLoopPolicy', 'portage.util.futures:compat_coroutine@_compat_coroutine', - 'portage.util._eventloop.EventLoop:EventLoop@_EventLoop', ) from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as _AsyncioEventLoop from portage.util._eventloop.global_event_loop import ( @@ -256,4 +255,4 @@ def _safe_loop(): """ if portage._internal_caller: return _global_event_loop() - return _EventLoop(main=False) + return _AsyncioEventLoop() -- 2.26.2
[gentoo-portage-dev] Re: [PATCH] Use default asyncio event loop implementation in API consumer threads
On 12/6/20 1:46 AM, Zac Medico wrote: > Make the _safe_loop function an alias for the global_event_loop > function, so that the default asyncio event loop implementation > will be used in API consumer threads. This is possible because > global_event_loop has been fixed (bug 758740) to always use > AsyncioEventLoop, and that uses asyncio.get_event_loop() which > returns a new event loop for each thread. I think we may still need a separate _safe_loop function here, since global_event_loop returns a separate loop per pid, but _safe_loop needs to return a separate loop per thread. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] Use default asyncio event loop implementation in API consumer threads
Make the _safe_loop function an alias for the global_event_loop function, so that the default asyncio event loop implementation will be used in API consumer threads. This is possible because global_event_loop has been fixed (bug 758740) to always use AsyncioEventLoop, and that uses asyncio.get_event_loop() which returns a new event loop for each thread. Bug: https://bugs.gentoo.org/758755 Signed-off-by: Zac Medico --- lib/portage/util/futures/_asyncio/__init__.py | 17 + 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index a902ad895..ce3685709 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -39,6 +39,7 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as _AsyncioEventLoop from portage.util._eventloop.global_event_loop import ( global_event_loop as _global_event_loop, + global_event_loop as _safe_loop, ) # pylint: disable=redefined-builtin from portage.util.futures.futures import ( @@ -241,19 +242,3 @@ def _wrap_loop(loop=None): loop = loop or _global_event_loop() return (loop if hasattr(loop, '_asyncio_wrapper') else _AsyncioEventLoop(loop=loop)) - - -def _safe_loop(): - """ - Return an event loop that's safe to use within the current context. - For portage internal callers, this returns a globally shared event - loop instance. For external API consumers, this constructs a - temporary event loop instance that's safe to use in a non-main - thread (it does not override the global SIGCHLD handler). - - @rtype: asyncio.AbstractEventLoop (or compatible) - @return: event loop instance - """ - if portage._internal_caller: - return _global_event_loop() - return _EventLoop(main=False) -- 2.26.2
[gentoo-portage-dev] [PATCH] Use default asyncio event loop implementation in child processes
Use the default asyncio event loop implementation in child
processes, instead of portage's internal EventLoop. After
fork, instantiate a new asyncio.DefaultEventLoopPolicy as
a workaround for https://bugs.python.org/issue22087, which
is necessary for RetryTestCase to succeed.
Bug: https://bugs.gentoo.org/758740
Signed-off-by: Zac Medico
---
lib/portage/__init__.py | 4
lib/portage/util/_eventloop/global_event_loop.py | 7 ---
2 files changed, 4 insertions(+), 7 deletions(-)
diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 4d4b590a8..2b821e81a 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -9,6 +9,7 @@ VERSION = "HEAD"
# ===
try:
+ import asyncio
import sys
import errno
if not hasattr(errno, 'ESTALE'):
@@ -373,6 +374,9 @@ class _ForkWatcher:
@staticmethod
def hook(_ForkWatcher):
_ForkWatcher.current_pid = _os.getpid()
+ # Force instantiation of a new event loop as a workaround for
+ # https://bugs.python.org/issue22087.
+ asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
_ForkWatcher.hook(_ForkWatcher)
diff --git a/lib/portage/util/_eventloop/global_event_loop.py
b/lib/portage/util/_eventloop/global_event_loop.py
index 21a1d1970..413011178 100644
--- a/lib/portage/util/_eventloop/global_event_loop.py
+++ b/lib/portage/util/_eventloop/global_event_loop.py
@@ -2,11 +2,8 @@
# Distributed under the terms of the GNU General Public License v2
import portage
-from .EventLoop import EventLoop
from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop
-
-_MAIN_PID = portage.getpid()
_instances = {}
@@ -22,10 +19,6 @@ def global_event_loop():
return instance
constructor = AsyncioEventLoop
- # If the default constructor doesn't support multiprocessing,
- # then multiprocessing constructor is used in subprocesses.
- if not constructor.supports_multiprocessing and pid != _MAIN_PID:
- constructor = EventLoop
# Use the _asyncio_wrapper attribute, so that unit tests can compare
# the reference to one retured from _wrap_loop(), since they should
--
2.26.2
[gentoo-portage-dev] [PATCH] _get_lock_fn: support multiprocessing spawn start method (bug 758230)
Ensure that _get_lock_fn arguments to multiprocessing.Process will
successfully pickle, as required by the spawn start method, which
is the default for macOS since Python 3.8.
Since file descriptors are not inherited unless the fork start
method is used, the subprocess should only try to close an
inherited file descriptor for the fork start method.
Bug: https://bugs.gentoo.org/758230
Signed-off-by: Zac Medico
---
lib/portage/locks.py | 36 +++-
1 file changed, 23 insertions(+), 13 deletions(-)
diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 1073343be..193045c03 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -44,18 +44,7 @@ def _get_lock_fn():
if _lock_fn is not None:
return _lock_fn
- def _test_lock(fd, lock_path):
- os.close(fd)
- try:
- with open(lock_path, 'a') as f:
- fcntl.lockf(f.fileno(),
fcntl.LOCK_EX|fcntl.LOCK_NB)
- except EnvironmentError as e:
- if e.errno == errno.EAGAIN:
- # Parent process holds lock, as expected.
- sys.exit(0)
- # Something went wrong.
- sys.exit(1)
fd, lock_path = tempfile.mkstemp()
try:
@@ -64,8 +53,16 @@ def _get_lock_fn():
except EnvironmentError:
pass
else:
- proc = multiprocessing.Process(target=_test_lock,
- args=(fd, lock_path))
+ proc = multiprocessing.Process(
+ target=_subprocess_test_lock,
+ args=(
+ # Since file descriptors are not
inherited unless the fork start
+ # method is used, the subprocess should
only try to close an
+ # inherited file descriptor for the
fork start method.
+ fd if
multiprocessing.get_start_method() == "fork" else None,
+ lock_path,
+ ),
+ )
proc.start()
proc.join()
if proc.exitcode == os.EX_OK:
@@ -80,6 +77,19 @@ def _get_lock_fn():
_lock_fn = fcntl.flock
return _lock_fn
+def _subprocess_test_lock(fd, lock_path):
+ if fd is not None:
+ os.close(fd)
+ try:
+ with open(lock_path, 'a') as f:
+ fcntl.lockf(f.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
+ except EnvironmentError as e:
+ if e.errno == errno.EAGAIN:
+ # Parent process holds lock, as expected.
+ sys.exit(0)
+
+ # Something went wrong.
+ sys.exit(1)
_open_fds = {}
_open_inodes = {}
--
2.26.2
[gentoo-portage-dev] [PATCH] find_smallest_cycle: don't merge satisfied PDEPEND too early
After PDEPENDs have been neglected by the find_smallest_cycle function,
do not try to merge them too early if they are already satisfied by
an installed package. This fixes incorrect merge order for PDEPEND
cycles involving xorg-server and xorg-drivers, which was triggered
by commit 5095c2023595a75e2848f1ad3dbe25b5fb451a44 because it gave
PDEPEND higher priority than satisfied buildtime dependencies.
Fixes: 5095c2023595 ("find_smallest_cycle: enhance search prioritization")
Reported-by: josef64 in #gentoo-portage
Bug: https://bugs.gentoo.org/754903
Signed-off-by: Zac Medico
---
lib/_emerge/DepPrioritySatisfiedRange.py | 1 +
lib/_emerge/depgraph.py | 8 +++---
.../tests/resolver/test_merge_order.py| 27 +--
3 files changed, 30 insertions(+), 6 deletions(-)
diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py
b/lib/_emerge/DepPrioritySatisfiedRange.py
index fb0d7db4e..f546590e0 100644
--- a/lib/_emerge/DepPrioritySatisfiedRange.py
+++ b/lib/_emerge/DepPrioritySatisfiedRange.py
@@ -93,6 +93,7 @@ class DepPrioritySatisfiedRange:
ignore_medium = _ignore_runtime
ignore_medium_soft = _ignore_satisfied_buildtime_slot_op
ignore_medium_post = _ignore_runtime_post
+ ignore_medium_post_satisifed = _ignore_satisfied_runtime_post
ignore_soft= _ignore_optional
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 1271bda3e..0450291d4 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -8052,18 +8052,18 @@ class depgraph:
(selected_nodes[0],), noiselevel=-1)
if selected_nodes and ignore_priority is not None:
- # Try to merge ignored medium_post deps as soon
as possible
+ # Try to merge neglected medium_post deps as
soon as possible
# if they're not satisfied by installed
packages.
for node in selected_nodes:
children =
set(mygraph.child_nodes(node))
- soft = children.difference(
+ medium_post_satisifed =
children.difference(
mygraph.child_nodes(node,
ignore_priority = \
-
DepPrioritySatisfiedRange.ignore_soft))
+
DepPrioritySatisfiedRange.ignore_medium_post_satisifed))
medium_post = children.difference(
mygraph.child_nodes(node,
ignore_priority=DepPrioritySatisfiedRange.ignore_medium_post))
- medium_post -= soft
+ medium_post -= medium_post_satisifed
for child in medium_post:
if child in selected_nodes:
continue
diff --git a/lib/portage/tests/resolver/test_merge_order.py
b/lib/portage/tests/resolver/test_merge_order.py
index f81fd2f6f..0510a0636 100644
--- a/lib/portage/tests/resolver/test_merge_order.py
+++ b/lib/portage/tests/resolver/test_merge_order.py
@@ -217,12 +217,23 @@ class MergeOrderTestCase(TestCase):
"IUSE" : "X +encode",
"RDEPEND" : "|| (
>=media-video/ffmpeg-0.6.90_rc0-r2[X=,encode=]
>=media-video/libav-0.6.90_rc[X=,encode=] )",
},
+ "x11-base/xorg-drivers-1.20-r2": {
+ "EAPI": "7",
+ "IUSE": "+video_cards_fbdev",
+ "PDEPEND": "x11-base/xorg-server
video_cards_fbdev? ( x11-drivers/xf86-video-fbdev )",
+ },
"x11-base/xorg-server-1.14.1" : {
"EAPI" : "5",
"SLOT": "0/1.14.1",
"DEPEND" : "media-libs/mesa",
"RDEPEND" : "media-libs/mesa",
+ "PDEPEND": "x11-base/xorg-drivers",
},
+ "x11-drivers/xf86-video-fbdev-0.5.0-r1": {
+ "EAPI": "7",
+ "DEPEND": "x11-base/xorg-server:=",
+
[gentoo-portage-dev] Re: [PATCH] Updates for portage-3.0.11 release
On 12/2/20 12:18 AM, Zac Medico wrote: > Signed-off-by: Zac Medico > --- > RELEASE-NOTES | 6 ++ > setup.py | 2 +- > 2 files changed, 7 insertions(+), 1 deletion(-) Please ignore. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] Updates for portage-3.0.11 release
Signed-off-by: Zac Medico --- RELEASE-NOTES | 6 ++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 61c3c3d16..7fff83017 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -1,6 +1,12 @@ Release Notes; upgrade information mainly. Features/major bugfixes are listed in NEWS +portage-3.0.11 +== +* Bug Fixes: +- Bug 756961 handle dev-lang/rust[system-bootstrap] dependency cycle +- Bug 757306 backtracking: fix virtual choices for circular deps + portage-3.0.10 == * Bug Fixes: diff --git a/setup.py b/setup.py index 443b94422..95a53307d 100755 --- a/setup.py +++ b/setup.py @@ -655,7 +655,7 @@ class build_ext(_build_ext): setup( name = 'portage', - version = '3.0.10', + version = '3.0.11', url = 'https://wiki.gentoo.org/wiki/Project:Portage', author = 'Gentoo Portage Development Team', author_email = 'dev-port...@gentoo.org', -- 2.26.2
[gentoo-portage-dev] [PATCH] Allow a package to replace its own buildtime dependency
If a package has a buildtime dependency on a previous version that
it will replace, then do not treat it as a slot conflict. This
solves inappropriate behavior for dev-lang/rust[system-bootstrap].
This requires adjustments to package selection logic in several
locations, in order to ensure that an installed package instance
will be selected to satisfy a buildtime dependency when
appropriate. Dependencies of the installed package will be
entirely ignored, but that has already been the case when using
installed package to break cycles, as discussed in bug 199856.
Bug: https://bugs.gentoo.org/756961
Signed-off-by: Zac Medico
---
lib/_emerge/depgraph.py | 68 ++
lib/portage/dep/dep_check.py | 24 ---
.../resolver/test_circular_choices_rust.py| 69 +++
3 files changed, 139 insertions(+), 22 deletions(-)
create mode 100644 lib/portage/tests/resolver/test_circular_choices_rust.py
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index d10474ab3..1271bda3e 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -85,6 +85,8 @@ from _emerge.resolver.output import Display,
format_unmatched_atom
# Exposes a depgraph interface to dep_check.
_dep_check_graph_interface =
collections.namedtuple('_dep_check_graph_interface',(
+ # Checks if parent package will replace child.
+ 'will_replace_child',
# Indicates a removal action, like depclean or prune.
'removal_action',
# Checks if update is desirable for a given package.
@@ -507,6 +509,7 @@ class _dynamic_depgraph_config:
# Track missed updates caused by solved conflicts.
self._conflict_missed_update = collections.defaultdict(dict)
dep_check_iface = _dep_check_graph_interface(
+ will_replace_child=depgraph._will_replace_child,
removal_action="remove" in myparams,
want_update_pkg=depgraph._want_update_pkg,
)
@@ -3104,6 +3107,22 @@ class depgraph:
self._frozen_config.myopts,
modified_use=self._pkg_use_enabled(pkg))),
level=logging.DEBUG, noiselevel=-1)
+ elif (pkg.installed and myparent and
+ pkg.root == myparent.root and
+ pkg.slot_atom == myparent.slot_atom):
+ # If the parent package is replacing
the child package then
+ # there's no slot conflict. Since the
child will be replaced,
+ # do not add it to the graph. No
attempt will be made to
+ # satisfy its dependencies, which is
unsafe if it has any
+ # missing dependencies, as discussed in
bug 199856.
+ if debug:
+ writemsg_level(
+ "%s%s %s\n" % ("Replace
Child:".ljust(15),
+ pkg,
pkg_use_display(pkg,
+
self._frozen_config.myopts,
+
modified_use=self._pkg_use_enabled(pkg))),
+ level=logging.DEBUG,
noiselevel=-1)
+ return 1
else:
if debug:
@@ -5877,6 +5896,27 @@ class depgraph:
(arg_atoms or update) and
not self._too_deep(depth))
+ def _will_replace_child(self, parent, root, atom):
+ """
+ Check if a given parent package will replace a child package
+ for the given root and atom.
+
+ @param parent: parent package
+ @type parent: Package
+ @param root: child root
+ @type root: str
+ @param atom: child atom
+ @type atom: Atom
+ @rtype: Package
+ @return: child package to replace, or None
+ """
+ if parent.root != root or parent.cp != atom.cp:
+ return None
+ for child in
self._iter_match_pkgs(self._frozen_config.roots[root], "installed", atom):
+ if parent.slot_atom == child.slot_atom:
+ return child
+ return None
+
def _too_deep(self, depth):
""
[gentoo-portage-dev] [PATCH] backtracking: fix virtual choices for circular deps (bug 757306)
Fix virtual choices to be consistent with circular dependency
backtracking choices.
Fixes: f78a91e44e3e ("backtracking: adjust || preference to break dependency
cycles")
Bug: https://bugs.gentoo.org/757306
Signed-off-by: Zac Medico
---
lib/portage/dep/dep_check.py | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dep/dep_check.py b/lib/portage/dep/dep_check.py
index 60c8ec6d0..b89d5d651 100644
--- a/lib/portage/dep/dep_check.py
+++ b/lib/portage/dep/dep_check.py
@@ -356,6 +356,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0,
trees=None,
# Alias the trees we'll be checking availability against
parent = trees[myroot].get("parent")
+ virt_parent = trees[myroot].get("virt_parent")
priority = trees[myroot].get("priority")
graph_db = trees[myroot].get("graph_db")
graph= trees[myroot].get("graph")
@@ -596,8 +597,10 @@ def dep_zapdeps(unreduced, reduced, myroot,
use_binaries=0, trees=None,
if
match_from_list(atom, cpv_slot_list):
circular_atom =
atom
break
- else:
- for circular_child in
circular_dependency.get(parent, []):
+ if circular_atom is None and
circular_dependency is not None:
+ for circular_child in itertools.chain(
+
circular_dependency.get(parent, []),
+
circular_dependency.get(virt_parent, [])):
for atom in
atoms:
if not
atom.blocker and atom.match(circular_child):
circular_atom = atom
--
2.26.2
[gentoo-portage-dev] [PATCH v2] find_smallest_cycle: enhance search prioritization
Enhance the find_smallest_cycle function to prioritize its search so that it will minimize the use of installed packages to break cycles. When installed packages must be used to break cycles, it will now prefer to do this for runtime dependencies over buildtime dependencies, since it's preferable to build against latest versions of buildtime dependencies whenever possible. This should solve some cases of bug 199856 which have been triggered by unsafe reliance on installed packages to break cycles. The included unit test case demonstrates correct merge order for a dependency calculation involving 6 independent cycles. This test case fails with the master branch, due to a buildtime dependency cycle of 3 packages being merged earlier than cycles of 2 packages. We can generalize this to say that the master branch may use an installed package to break an arbitrarily sized cycle in a somewhat random location, even though that cycle may be composed of smaller independent cycles which would be safer to break individually. Bug: https://bugs.gentoo.org/754903 Signed-off-by: Zac Medico --- [PATCH v2] * Add a unit test case which demonstrates a significant flaw in the master branch. * Sort nodes in find_smallest_cycle, for deterministic results. lib/_emerge/DepPriorityNormalRange.py | 2 + lib/_emerge/DepPrioritySatisfiedRange.py | 52 ++- lib/_emerge/depgraph.py | 43 +-- .../tests/resolver/test_merge_order.py| 10 4 files changed, 66 insertions(+), 41 deletions(-) diff --git a/lib/_emerge/DepPriorityNormalRange.py b/lib/_emerge/DepPriorityNormalRange.py index 5f3f3da70..10f205a3b 100644 --- a/lib/_emerge/DepPriorityNormalRange.py +++ b/lib/_emerge/DepPriorityNormalRange.py @@ -14,6 +14,7 @@ class DepPriorityNormalRange: """ MEDIUM = 3 MEDIUM_SOFT = 2 + MEDIUM_POST = 2 SOFT= 1 NONE= 0 @@ -37,6 +38,7 @@ class DepPriorityNormalRange: ignore_medium = _ignore_runtime ignore_medium_soft = _ignore_runtime_post + ignore_medium_post = _ignore_runtime_post ignore_soft= _ignore_optional DepPriorityNormalRange.ignore_priority = ( diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py b/lib/_emerge/DepPrioritySatisfiedRange.py index e056e676f..df2439f1f 100644 --- a/lib/_emerge/DepPrioritySatisfiedRange.py +++ b/lib/_emerge/DepPrioritySatisfiedRange.py @@ -8,17 +8,18 @@ class DepPrioritySatisfiedRange: not satisfied and buildtimeHARD not satisfied and runtime 7 MEDIUM - not satisfied and runtime_post 6 MEDIUM_SOFT - satisfied and buildtime_slot_op5 SOFT - satisfied and buildtime4 SOFT - satisfied and runtime 3 SOFT - satisfied and runtime_post 2 SOFT + satisfied and buildtime_slot_op6 MEDIUM_SOFT + satisfied and buildtime5 MEDIUM_SOFT + satisfied and runtime 4 MEDIUM_SOFT + runtime_post 3 MEDIUM_POST + satisfied and runtime_post 2 MEDIUM_POST optional 1 SOFT (none of the above)0 NONE """ MEDIUM = 7 MEDIUM_SOFT = 6 - SOFT= 5 + MEDIUM_POST = 3 + SOFT= 1 NONE= 0 @classmethod @@ -37,15 +38,23 @@ class DepPrioritySatisfiedRange: return False return bool(priority.runtime_post) + @classmethod + def _ignore_runtime_post(cls, priority): + if priority.__class__ is not DepPriority: + return False + return bool(priority.optional or priority.runtime_post) + @classmethod def _ignore_satisfied_runtime(cls, priority): if priority.__class__ is not DepPriority: return False if priority.optional: return True - if not priority.satisfied: + if priority.buildtime: return False - return not priority.buildtime + if not priority.runtime: + return True + return bool(priority.satisfied) @classmethod def _ignore_satisfied_buildtime(cls, priority): @@ -61,37 +70,32 @@ class DepPrioritySatisfiedRange: def _ignore_satisfied_buildtime_slot_op(cls, priority): if priority.__class__ is not DepPriority: return False - return bool(priority.optional or \ - priority.satisfied) - - @classmethod - def _ignore_runtime_post(cls, priority
[gentoo-portage-dev] [PATCH] find_smallest_cycle: enhance search prioritization
Enhance the find_smallest_cycle function to prioritize its search so that it will minimize the use of installed packages to break cycles. When installed packages must be used to break cycles, it will now prefer to do this for runtime dependencies over buildtime dependencies, since it's preferable to build against latest versions of buildtime dependencies whenever possible. This should solve some cases of bug 199856 which have been triggered by unsafe reliance on installed packages to break cycles. Bug: https://bugs.gentoo.org/754903 Signed-off-by: Zac Medico --- lib/_emerge/DepPriorityNormalRange.py| 2 + lib/_emerge/DepPrioritySatisfiedRange.py | 52 +--- lib/_emerge/depgraph.py | 41 +++ 3 files changed, 54 insertions(+), 41 deletions(-) diff --git a/lib/_emerge/DepPriorityNormalRange.py b/lib/_emerge/DepPriorityNormalRange.py index 5f3f3da70..10f205a3b 100644 --- a/lib/_emerge/DepPriorityNormalRange.py +++ b/lib/_emerge/DepPriorityNormalRange.py @@ -14,6 +14,7 @@ class DepPriorityNormalRange: """ MEDIUM = 3 MEDIUM_SOFT = 2 + MEDIUM_POST = 2 SOFT= 1 NONE= 0 @@ -37,6 +38,7 @@ class DepPriorityNormalRange: ignore_medium = _ignore_runtime ignore_medium_soft = _ignore_runtime_post + ignore_medium_post = _ignore_runtime_post ignore_soft= _ignore_optional DepPriorityNormalRange.ignore_priority = ( diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py b/lib/_emerge/DepPrioritySatisfiedRange.py index e056e676f..df2439f1f 100644 --- a/lib/_emerge/DepPrioritySatisfiedRange.py +++ b/lib/_emerge/DepPrioritySatisfiedRange.py @@ -8,17 +8,18 @@ class DepPrioritySatisfiedRange: not satisfied and buildtimeHARD not satisfied and runtime 7 MEDIUM - not satisfied and runtime_post 6 MEDIUM_SOFT - satisfied and buildtime_slot_op5 SOFT - satisfied and buildtime4 SOFT - satisfied and runtime 3 SOFT - satisfied and runtime_post 2 SOFT + satisfied and buildtime_slot_op6 MEDIUM_SOFT + satisfied and buildtime5 MEDIUM_SOFT + satisfied and runtime 4 MEDIUM_SOFT + runtime_post 3 MEDIUM_POST + satisfied and runtime_post 2 MEDIUM_POST optional 1 SOFT (none of the above)0 NONE """ MEDIUM = 7 MEDIUM_SOFT = 6 - SOFT= 5 + MEDIUM_POST = 3 + SOFT= 1 NONE= 0 @classmethod @@ -37,15 +38,23 @@ class DepPrioritySatisfiedRange: return False return bool(priority.runtime_post) + @classmethod + def _ignore_runtime_post(cls, priority): + if priority.__class__ is not DepPriority: + return False + return bool(priority.optional or priority.runtime_post) + @classmethod def _ignore_satisfied_runtime(cls, priority): if priority.__class__ is not DepPriority: return False if priority.optional: return True - if not priority.satisfied: + if priority.buildtime: return False - return not priority.buildtime + if not priority.runtime: + return True + return bool(priority.satisfied) @classmethod def _ignore_satisfied_buildtime(cls, priority): @@ -61,37 +70,32 @@ class DepPrioritySatisfiedRange: def _ignore_satisfied_buildtime_slot_op(cls, priority): if priority.__class__ is not DepPriority: return False - return bool(priority.optional or \ - priority.satisfied) - - @classmethod - def _ignore_runtime_post(cls, priority): - if priority.__class__ is not DepPriority: - return False - return bool(priority.optional or \ - priority.satisfied or \ - priority.runtime_post) + if priority.optional: + return True + if priority.satisfied: + return True + return not priority.buildtime and not priority.runtime @classmethod def _ignore_runtime(cls, priority): if priority.__class__ is not DepPriority: return False - return bool(priority.satisfied or \ - priority.optional or \ -
[gentoo-portage-dev] [PATCH] emerge: Disable profile deprecation warning inheritance (bug 753497)
According to PMS, a deprecated profile warning is not inherited. Since
the current profile node may have been inherited by a user profile
node, the deprecation warning may be relevant even if it is not a
top-level profile node. Therefore, consider the deprecated warning
to be irrelevant when the current profile node belongs to the same
repo as the previous profile node.
Bug: https://bugs.gentoo.org/753497
Signed-off-by: Zac Medico
---
.../ebuild/_config/LocationsManager.py| 30 ++-
.../ebuild/deprecated_profile_check.py| 9 +++---
2 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/lib/portage/package/ebuild/_config/LocationsManager.py
b/lib/portage/package/ebuild/_config/LocationsManager.py
index b90b9227c..28740b355 100644
--- a/lib/portage/package/ebuild/_config/LocationsManager.py
+++ b/lib/portage/package/ebuild/_config/LocationsManager.py
@@ -30,7 +30,9 @@ _PORTAGE1_DIRECTORIES = frozenset([
_profile_node = collections.namedtuple('_profile_node',
('location', 'portage1_directories', 'user_config',
- 'profile_formats', 'eapi', 'allow_build_id'))
+ 'profile_formats', 'eapi', 'allow_build_id',
+ 'show_deprecated_warning',
+))
_allow_parent_colon = frozenset(
["portage-2"])
@@ -132,7 +134,7 @@ class LocationsManager:
if self.profile_path:
try:
self._addProfile(os.path.realpath(self.profile_path),
- repositories, known_repos)
+ repositories, known_repos, ())
except ParseError as e:
if not portage._sync_mode:
writemsg(_("!!! Unable to parse
profile: '%s'\n") % self.profile_path, noiselevel=-1)
@@ -154,7 +156,9 @@ class LocationsManager:
('profile-bashrcs', 'profile-set'),
read_corresponding_eapi_file(
custom_prof + os.sep, default=None),
- True))
+ True,
+ show_deprecated_warning=False,
+ ))
del custom_prof
self.profiles = tuple(self.profiles)
@@ -167,7 +171,7 @@ class LocationsManager:
noiselevel=-1)
raise DirectoryNotFound(var)
- def _addProfile(self, currentPath, repositories, known_repos):
+ def _addProfile(self, currentPath, repositories, known_repos,
previous_repos):
current_abs_path = os.path.abspath(currentPath)
allow_directories = True
allow_parent_colon = True
@@ -176,8 +180,8 @@ class LocationsManager:
current_formats = ()
eapi = None
- intersecting_repos = [x for x in known_repos
- if current_abs_path.startswith(x[0])]
+ intersecting_repos = tuple(x for x in known_repos
+ if current_abs_path.startswith(x[0]))
if intersecting_repos:
# Handle nested repositories. The longest path
# will be the correct one.
@@ -214,6 +218,14 @@ class LocationsManager:
for x in layout_data['profile-formats'])
current_formats = tuple(layout_data['profile-formats'])
+ # According to PMS, a deprecated profile warning is not
inherited. Since
+ # the current profile node may have been inherited by a user
profile
+ # node, the deprecation warning may be relevant even if it is
not a
+ # top-level profile node. Therefore, consider the deprecated
warning
+ # to be irrelevant when the current profile node belongs to the
same
+ # repo as the previous profile node.
+ show_deprecated_warning = (not previous_repos or
+ tuple(x[0] for x in previous_repos) != tuple(x[0] for x
in intersecting_repos))
if compat_mode:
offenders =
_PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath))
@@ -256,7 +268,7 @@ class LocationsManager:
parentPath =
os.path.realpath(parentPath)
if exists_raise_eaccess(parentPath):
- self._addProfile(parentPath,
repositories, known_repos)
+ self._addProfile(parentPath,
repositories, known_repos, intersecting_repos)
else:
raise ParseError(
_("Parent '%s' not found:
'%s'")
[gentoo-portage-dev] [PATCH] make.conf: expand special *ROOT variables (bug 752147)
Bug: https://bugs.gentoo.org/752147
Signed-off-by: Zac Medico
---
lib/portage/package/ebuild/config.py | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/portage/package/ebuild/config.py
b/lib/portage/package/ebuild/config.py
index a09fdbced..3be8f9f6d 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -401,9 +401,14 @@ class config:
expand_map = env_d.copy()
self._expand_map = expand_map
- # Allow make.globals to set default paths relative to
${EPREFIX}.
+ # Allow make.globals and make.conf to set default paths
relative to vars like ${EPREFIX}.
+ expand_map["BROOT"] = broot
expand_map["EPREFIX"] = eprefix
+ expand_map["EROOT"] = eroot
+ expand_map["ESYSROOT"] = esysroot
expand_map["PORTAGE_CONFIGROOT"] = config_root
+ expand_map["ROOT"] = target_root
+ expand_map["SYSROOT"] = sysroot
if portage._not_installed:
make_globals_path =
os.path.join(PORTAGE_BASE_PATH, "cnf", "make.globals")
--
2.26.2
[gentoo-portage-dev] [PATCH] emerge: add --quickpkg-direct-root option
Specify the root to use as the --quickpkg-direct package source. This
root is assumed to be immutable during the entire emerge operation.
The default is set to "/".
Bug: https://bugs.gentoo.org/752066
Signed-off-by: Zac Medico
---
lib/_emerge/actions.py | 19 ---
lib/_emerge/depgraph.py | 11 +--
lib/_emerge/main.py | 5 +
lib/portage/tests/emerge/test_simple.py | 3 ++-
man/emerge.1| 10 --
5 files changed, 40 insertions(+), 8 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 5e8a46957..2e155899c 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -49,7 +49,7 @@ from portage.package.ebuild._ipc.QueryCommand import
QueryCommand
from portage.package.ebuild.fetch import _hide_url_passwd
from portage._sets import load_default_config, SETPREFIX
from portage._sets.base import InternalPackageSet
-from portage.util import cmp_sort_key, writemsg, varexpand, \
+from portage.util import cmp_sort_key, normalize_path, writemsg, varexpand, \
writemsg_level, writemsg_stdout
from portage.util.digraph import digraph
from portage.util.SlotObject import SlotObject
@@ -106,13 +106,26 @@ def action_build(emerge_config, trees=DeprecationWarning,
# before we get here, so warn if they're not (bug #267103).
chk_updated_cfg_files(settings['EROOT'], ['/etc/portage'])
+ quickpkg_root = normalize_path(os.path.abspath(
+ emerge_config.opts.get('--quickpkg-direct-root',
+
emerge_config.running_config.settings['ROOT']))).rstrip(os.path.sep) +
os.path.sep
quickpkg_direct = ("--usepkg" in emerge_config.opts and
emerge_config.opts.get('--quickpkg-direct', 'n') == 'y' and
- emerge_config.target_config is not emerge_config.running_config)
+ emerge_config.target_config.settings['ROOT'] != quickpkg_root)
if '--getbinpkg' in emerge_config.opts or quickpkg_direct:
kwargs = {}
if quickpkg_direct:
- kwargs['add_repos'] =
(emerge_config.running_config.trees['vartree'].dbapi,)
+ if quickpkg_root == emerge_config.running_config.root:
+ quickpkg_vardb =
emerge_config.running_config.trees['vartree'].dbapi
+ else:
+ quickpkg_settings = portage.config(
+
config_root=emerge_config.target_config.settings['PORTAGE_CONFIGROOT'],
+ target_root=quickpkg_root,
+
env=emerge_config.target_config.settings.backupenv,
+
sysroot=emerge_config.target_config.settings['SYSROOT'],
+
eprefix=emerge_config.target_config.settings['EPREFIX'])
+ quickpkg_vardb =
portage.vartree(settings=quickpkg_settings).dbapi
+ kwargs['add_repos'] = (quickpkg_vardb,)
try:
emerge_config.target_config.trees['bintree'].populate(
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 0bb0352e7..898cf6c1a 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -41,7 +41,7 @@ from portage._sets import SETPREFIX
from portage._sets.base import InternalPackageSet
from portage.util import ConfigProtect, shlex_split, new_protect_filename
from portage.util import cmp_sort_key, writemsg, writemsg_stdout
-from portage.util import ensure_dirs
+from portage.util import ensure_dirs, normalize_path
from portage.util import writemsg_level, write_atomic
from portage.util.digraph import digraph
from portage.util.futures import asyncio
@@ -4567,8 +4567,15 @@ class depgraph:
self._dynamic_config._skip_restart = True
return False, myfavorites
+ # Since --quickpkg-direct assumes that --quickpkg-direct-root is
+ # immutable, assert that there are no merged or unmerge tasks
+ # for --quickpkg-direct-root.
+ quickpkg_root = normalize_path(os.path.abspath(
+ self._frozen_config.myopts.get('--quickpkg-direct-root',
+
self._frozen_config._running_root.settings['ROOT']))).rstrip(os.path.sep) +
os.path.sep
if (self._frozen_config.myopts.get('--quickpkg-direct', 'n') ==
'y' and
- self._frozen_config.target_root is not
self._frozen_config._running_root):
+ self._frozen_config.settings['ROOT'] != quickpkg_root
and
+ self._frozen_config._running_root.settings['ROOT'] ==
quickpkg_root):
running_root = self._frozen_config._running_root.root
[gentoo-portage-dev] [PATCH] pid-sandbox: Forward SIGTSTP and SIGCONT (bug 704498)
For correct operation of Ctrl+Z, forward SIGTSTP and SIGCONT
to all sandboxed pids.
Fixes: 37e4dc5ae842 ("pid-sandbox: pid-ns-init setsid support (bug 675870)")
Bug: https://bugs.gentoo.org/704498
Signed-off-by: Zac Medico
---
bin/pid-ns-init | 24 +++-
1 file changed, 23 insertions(+), 1 deletion(-)
diff --git a/bin/pid-ns-init b/bin/pid-ns-init
index 3a218a5df..e410dd028 100644
--- a/bin/pid-ns-init
+++ b/bin/pid-ns-init
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Copyright 2018-2019 Gentoo Authors
+# Copyright 2018-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -19,6 +19,11 @@ KILL_SIGNALS = (
signal.SIGHUP,
)
+SIGTSTP_SIGCONT = (
+ signal.SIGTSTP,
+ signal.SIGCONT,
+)
+
def forward_kill_signal(pid, signum, frame):
if pid == 0:
@@ -28,6 +33,18 @@ def forward_kill_signal(pid, signum, frame):
os.kill(pid, signum)
+def forward_sigtstp_sigcont(pid, signum, frame):
+ handler = None
+ if pid == 0:
+ # Temporarily disable the handler in order to prevent it from
+ # being called recursively, since the signal will also be sent
+ # to the current process.
+ handler = signal.signal(signum, signal.SIG_DFL)
+ os.kill(pid, signum)
+ if handler is not None:
+ signal.signal(signum, handler)
+
+
def preexec_fn(uid, gid, groups, umask):
if gid is not None:
os.setgid(gid)
@@ -97,6 +114,11 @@ def main(argv):
for signum in KILL_SIGNALS:
signal.signal(signum, sig_handler)
+ # For correct operation of Ctrl+Z, forward SIGTSTP and SIGCONT.
+ sigtstp_sigcont_handler = functools.partial(forward_sigtstp_sigcont, 0
if setsid else main_child_pid)
+ for signum in SIGTSTP_SIGCONT:
+ signal.signal(signum, sigtstp_sigcont_handler)
+
# wait for child processes
while True:
try:
--
2.26.2
[gentoo-portage-dev] [PATCH v2] emerge: enable parallel-fetch during pkg_pretend (bug 710432)
Execute pkg_pretend phases in a coroutine while parallel-fetch
is running concurrently. When it's time to execute the pkg_pretend
phase for a remote binary package, use a Scheduler _get_prefetcher
method to get a running prefetcher if available, and otherwise
start a new fetcher.
Since pkg_pretend phases now run inside of the --keep-going retry
loop, --keep-going is now able to recover from pkg_pretend
failures, which fixes bug 404157.
Bug: https://bugs.gentoo.org/404157
Bug: https://bugs.gentoo.org/710432
Signed-off-by: Zac Medico
---
[PATCH v2] records failed packages for correct interaction with
emerge --keep-going, which fixes bug 404157
lib/_emerge/Scheduler.py | 142 +++
1 file changed, 99 insertions(+), 43 deletions(-)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index a69421288..465f928a0 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -25,6 +25,7 @@ from portage._sets import SETPREFIX
from portage._sets.base import InternalPackageSet
from portage.util import ensure_dirs, writemsg, writemsg_level
from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
from portage.util.SlotObject import SlotObject
from portage.util._async.SchedulerInterface import SchedulerInterface
from portage.package.ebuild.digestcheck import digestcheck
@@ -766,7 +767,8 @@ class Scheduler(PollScheduler):
return prefetcher
- def _run_pkg_pretend(self):
+ @coroutine
+ def _run_pkg_pretend(self, loop=None):
"""
Since pkg_pretend output may be important, this method sends all
output directly to stdout (regardless of options like --quiet or
@@ -774,7 +776,7 @@ class Scheduler(PollScheduler):
"""
failures = 0
- sched_iface = self._sched_iface
+ sched_iface = loop = asyncio._wrap_loop(loop or
self._sched_iface)
for x in self._mergelist:
if not isinstance(x, Package):
@@ -789,18 +791,28 @@ class Scheduler(PollScheduler):
if "pretend" not in x.defined_phases:
continue
- out_str =">>> Running pre-merge checks for " +
colorize("INFORM", x.cpv) + "\n"
- portage.util.writemsg_stdout(out_str, noiselevel=-1)
+ out_str = "Running pre-merge checks for " +
colorize("INFORM", x.cpv)
+ self._status_msg(out_str)
root_config = x.root_config
- settings = self.pkgsettings[root_config.root]
+ settings = self._allocate_config(root_config.root)
settings.setcpv(x)
+ if not x.built:
+ # Get required SRC_URI metadata (it's not
cached in x.metadata
+ # because some packages have an extremely large
SRC_URI value).
+ portdb = root_config.trees["porttree"].dbapi
+ (settings.configdict["pkg"]["SRC_URI"],) =
yield portdb.async_aux_get(
+ x.cpv, ["SRC_URI"], myrepo=x.repo,
loop=loop
+ )
# setcpv/package.env allows for per-package
PORTAGE_TMPDIR so we
# have to validate it for each package
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
- return rval
+ failures += 1
+ self._record_pkg_failure(x, settings, FAILURE)
+ self._deallocate_config(settings)
+ continue
build_dir_path = os.path.join(
os.path.realpath(settings["PORTAGE_TMPDIR"]),
@@ -809,7 +821,7 @@ class Scheduler(PollScheduler):
settings["PORTAGE_BUILDDIR"] = build_dir_path
build_dir = EbuildBuildDir(scheduler=sched_iface,
settings=settings)
- sched_iface.run_until_complete(build_dir.async_lock())
+ yield build_dir.async_lock()
current_task = None
try:
@@ -835,7 +847,7 @@ class Scheduler(PollScheduler):
phase='clean',
scheduler=sched_iface, settings=settings)
current_task = clean_phase
clean_phase.start()
-
[gentoo-portage-dev] [PATCH] emerge: enable parallel-fetch during pkg_pretend (bug 710432)
Execute pkg_pretend phases in a coroutine while parallel-fetch is running concurrently. When it's time to execute the pkg_pretend phase for a remote binary package, use a Scheduler _get_prefetcher method to get a running prefetcher if available, and otherwise start a new fetcher. Bug: https://bugs.gentoo.org/710432 Signed-off-by: Zac Medico --- lib/_emerge/Scheduler.py | 94 +--- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index a69421288..20884986f 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -25,6 +25,7 @@ from portage._sets import SETPREFIX from portage._sets.base import InternalPackageSet from portage.util import ensure_dirs, writemsg, writemsg_level from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine, coroutine_return from portage.util.SlotObject import SlotObject from portage.util._async.SchedulerInterface import SchedulerInterface from portage.package.ebuild.digestcheck import digestcheck @@ -766,7 +767,8 @@ class Scheduler(PollScheduler): return prefetcher - def _run_pkg_pretend(self): + @coroutine + def _run_pkg_pretend(self, loop=None): """ Since pkg_pretend output may be important, this method sends all output directly to stdout (regardless of options like --quiet or @@ -774,7 +776,7 @@ class Scheduler(PollScheduler): """ failures = 0 - sched_iface = self._sched_iface + sched_iface = loop = asyncio._wrap_loop(loop or self._sched_iface) for x in self._mergelist: if not isinstance(x, Package): @@ -795,12 +797,18 @@ class Scheduler(PollScheduler): root_config = x.root_config settings = self.pkgsettings[root_config.root] settings.setcpv(x) + if not x.built: + # Get required SRC_URI metadata (it's not cached in x.metadata + # because some packages have an extremely large SRC_URI value). + portdb = root_config.trees["porttree"].dbapi + settings.configdict["pkg"]["SRC_URI"], = (yield portdb.async_aux_get( + x.cpv, ["SRC_URI"], myrepo=x.repo, loop=loop)) # setcpv/package.env allows for per-package PORTAGE_TMPDIR so we # have to validate it for each package rval = _check_temp_dir(settings) if rval != os.EX_OK: - return rval + coroutine_return(rval) build_dir_path = os.path.join( os.path.realpath(settings["PORTAGE_TMPDIR"]), @@ -809,7 +817,7 @@ class Scheduler(PollScheduler): settings["PORTAGE_BUILDDIR"] = build_dir_path build_dir = EbuildBuildDir(scheduler=sched_iface, settings=settings) - sched_iface.run_until_complete(build_dir.async_lock()) + yield build_dir.async_lock() current_task = None try: @@ -835,7 +843,7 @@ class Scheduler(PollScheduler): phase='clean', scheduler=sched_iface, settings=settings) current_task = clean_phase clean_phase.start() - clean_phase.wait() + yield clean_phase.async_wait() if x.built: tree = "bintree" @@ -845,10 +853,11 @@ class Scheduler(PollScheduler): # Display fetch on stdout, so that it's always clear what # is consuming time here. if bintree.isremote(x.cpv): - fetcher = BinpkgFetcher(pkg=x, - scheduler=sched_iface) - fetcher.start() - if fetcher.wait() != os.EX_OK: + fetcher = self._get_prefetcher(x) + if fetcher is None: + fetcher = BinpkgFetcher(pkg=x, scheduler=loop) +
[gentoo-portage-dev] [PATCH] _slot_confict_backtrack: group similar missed updates (bug 743115)
When a slot conflict occurs due to a missed update, and some other
similar update(s) are available, add the similar update(s) to the
runtime package mask for the same backtracking choice. This reduces
minimum number of backtrack tries required to solve the test case
for bug 743115 from 7 to 4, where the difference of 3 corresponds
to the number of other similar setuptools updates available.
Bug: https://bugs.gentoo.org/743115
Signed-off-by: Zac Medico
---
lib/_emerge/depgraph.py | 25 ---
lib/_emerge/resolver/backtracking.py | 7 +++---
.../test_slot_operator_missed_update.py | 2 +-
3 files changed, 26 insertions(+), 8 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 7281d8692..2a840b2ca 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1795,15 +1795,32 @@ class depgraph:
self._dynamic_config._parent_atoms.get(to_be_masked, set())
conflict_atoms = set(parent_atom for parent_atom in
all_parents \
if parent_atom not in parent_atoms)
- backtrack_data.append((to_be_masked, conflict_atoms))
+
+ similar_pkgs = []
+ if conflict_atoms:
+ # If the conflict has been triggered by a
missed update, then
+ # we can avoid excessive backtracking if we
detect similar missed
+ # updates and mask them as part of the same
backtracking choice.
+ for similar_pkg in
self._iter_similar_available(to_be_masked, slot_atom):
+ if similar_pkg in conflict_pkgs:
+ continue
+ similar_conflict_atoms = []
+ for parent_atom in conflict_atoms:
+ parent, atom = parent_atom
+ if not atom.match(similar_pkg):
+
similar_conflict_atoms.append(parent_atom)
+ if similar_conflict_atoms:
+
similar_pkgs.append((similar_pkg, set(similar_conflict_atoms)))
+ similar_pkgs.append((to_be_masked, conflict_atoms))
+ backtrack_data.append(tuple(similar_pkgs))
# Prefer choices that minimize conflict atoms. This is intended
# to take precedence over the earlier package version sort. The
# package version sort is still needed or else choices for the
# testOverlapSlotConflict method of
VirtualMinimizeChildrenTestCase
# become non-deterministic.
- backtrack_data.sort(key=lambda item: len(item[1]))
- to_be_masked = backtrack_data[-1][0]
+ backtrack_data.sort(key=lambda similar_pkgs: max(len(item[1])
for item in similar_pkgs))
+ to_be_masked = [item[0] for item in backtrack_data[-1]]
self._dynamic_config._backtrack_infos.setdefault(
"slot conflict", []).append(backtrack_data)
@@ -1814,7 +1831,7 @@ class depgraph:
"",
"backtracking due to slot conflict:",
" first package: %s" % existing_node,
- " package to mask: %s" % to_be_masked,
+ " package(s) to mask: %s" % str(to_be_masked),
" slot: %s" % slot_atom,
" parents: %s" % ", ".join(
"(%s, '%s')" % (ppkg, atom) for ppkg,
atom in all_parents
diff --git a/lib/_emerge/resolver/backtracking.py
b/lib/_emerge/resolver/backtracking.py
index bc3fb3206..ca94623ac 100644
--- a/lib/_emerge/resolver/backtracking.py
+++ b/lib/_emerge/resolver/backtracking.py
@@ -166,13 +166,14 @@ class Backtracker:
self._feedback_slot_conflict(conflicts_data[0])
def _feedback_slot_conflict(self, conflict_data):
- for pkg, parent_atoms in conflict_data:
+ for similar_pkgs in conflict_data:
new_node = copy.deepcopy(self._current_node)
new_node.depth += 1
new_node.mask_steps += 1
new_node.terminal = False
- new_node.parameter.runtime_pkg_mask.setdefault(
- pkg, {})["slot conflict"] = parent_atoms
+ for pkg, parent_atoms in similar_pkgs:
+
