[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
commit: 6f9a10d38259dd61b948837e193b047464791845
Author: Zac Medico gentoo org>
AuthorDate: Mon Mar 4 05:31:08 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Mar 4 05:33:44 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6f9a10d3
SpawnProcess: Optimize away null input for create_pipe=False
When create_pipe=False support was added in commit e8b31c86eaed,
a null input file descriptor was used for PipeLogger and BuildLogger
instances. Optimize this away, eliminating the unnecessary loggers.
Fixes: e8b31c86eaed ("ForkProcess: Prevent redundant pipe and set_term_size
recursion")
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/SpawnProcess.py| 51 +-
lib/portage/util/_async/ForkProcess.py | 7 ++---
2 files changed, 28 insertions(+), 30 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 9fc12c42e5..513a7b2fe4 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -79,10 +79,7 @@ class SpawnProcess(SubProcess):
# SpawnProcess will have created a pipe earlier, so it
# would be redundant to do it here (it could also trigger
# spawn recursion via set_term_size as in bug 923750).
-# Use /dev/null for master_fd, triggering early return
-# of _main, followed by _async_waitpid.
-# TODO: Optimize away the need for master_fd here.
-master_fd = os.open(os.devnull, os.O_RDONLY)
+master_fd = None
slave_fd = None
can_log = False
@@ -166,23 +163,27 @@ class SpawnProcess(SubProcess):
self._registered = True
def _start_main_task(self, pr, log_file_path=None, stdout_fd=None):
-build_logger = BuildLogger(
-env=self.env,
-log_path=log_file_path,
-log_filter_file=self.log_filter_file,
-scheduler=self.scheduler,
-)
-build_logger.start()
-
-pipe_logger = PipeLogger(
-background=self.background,
-scheduler=self.scheduler,
-input_fd=pr,
-log_file_path=build_logger.stdin,
-stdout_fd=stdout_fd,
-)
-
-pipe_logger.start()
+if pr is None:
+build_logger = None
+pipe_logger = None
+else:
+build_logger = BuildLogger(
+env=self.env,
+log_path=log_file_path,
+log_filter_file=self.log_filter_file,
+scheduler=self.scheduler,
+)
+build_logger.start()
+
+pipe_logger = PipeLogger(
+background=self.background,
+scheduler=self.scheduler,
+input_fd=pr,
+log_file_path=build_logger.stdin,
+stdout_fd=stdout_fd,
+)
+
+pipe_logger.start()
self._main_task_cancel = functools.partial(
self._main_cancel, build_logger, pipe_logger
@@ -198,18 +199,18 @@ class SpawnProcess(SubProcess):
await self._pty_ready
self._pty_ready = None
try:
-if pipe_logger.poll() is None:
+if pipe_logger is not None and pipe_logger.poll() is None:
await pipe_logger.async_wait()
-if build_logger.poll() is None:
+if build_logger is not None and build_logger.poll() is None:
await build_logger.async_wait()
except asyncio.CancelledError:
self._main_cancel(build_logger, pipe_logger)
raise
def _main_cancel(self, build_logger, pipe_logger):
-if pipe_logger.poll() is None:
+if pipe_logger is not None and pipe_logger.poll() is None:
pipe_logger.cancel()
-if build_logger.poll() is None:
+if build_logger is not None and build_logger.poll() is None:
build_logger.cancel()
def _main_exit(self, main_task):
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index ebcbd94107..e6cfdefb88 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -91,11 +91,8 @@ class ForkProcess(SpawnProcess):
# When called via process.spawn, SpawnProcess
# will have created a pipe earlier, so it would be
# redundant to do it here (it could also trigger spawn
-# recursion via set_term_size as in bug 923750). Use
-# /dev/null for master_fd, triggering early return
-# of _main, followed by _async_waitpid.
-# TODO: Optimize away the need for master_fd here.
-master_fd = os.open(os.devnull, os.O_RDONLY)
+# recursion via
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/dbapi/, lib/portage/util/_async/, ...
commit: acb69a6f234bd412e95e76f5c1db1b1f5b8e1dc5
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 24 03:52:47 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 24 04:23:27 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=acb69a6f
SchedulerInterface/PollScheduler: Add _loop property
This allows async_aux_get to easily verify the identity of
the underlying loop so that this assertion will not fail:
_start_with_metadata
(settings.configdict["pkg"]["SRC_URI"],) = aux_get_task.future.result()
File "/usr/lib/python3.11/site-packages/portage/dbapi/porttree.py", line 786,
in async_aux_get
raise AssertionError(
AssertionError: async_aux_get called from thread <_MainThread(MainThread,
started 281473559502880)> with loop <_emerge.Scheduler.Scheduler._iface_class
object at 0x8e3a8840>
Terminated
Fixes: 389bb304abf5 ("async_aux_get: Use EbuildMetadataPhase deallocate_config
future")
Bug: https://bugs.gentoo.org/925333
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/PollScheduler.py| 9 -
lib/portage/dbapi/porttree.py | 2 +-
lib/portage/tests/ebuild/test_doebuild_spawn.py | 3 ++-
lib/portage/tests/ebuild/test_ipc_daemon.py | 3 ++-
lib/portage/util/_async/SchedulerInterface.py | 9 -
5 files changed, 21 insertions(+), 5 deletions(-)
diff --git a/lib/_emerge/PollScheduler.py b/lib/_emerge/PollScheduler.py
index e5dffd8afa..b5bfd20b76 100644
--- a/lib/_emerge/PollScheduler.py
+++ b/lib/_emerge/PollScheduler.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import threading
@@ -39,6 +39,13 @@ class PollScheduler:
self._event_loop, is_background=self._is_background
)
+@property
+def _loop(self):
+"""
+Returns the real underlying asyncio loop.
+"""
+return self._event_loop._loop
+
def _is_background(self):
return self._background
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 4eebe1183f..de6aa5c82b 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -775,7 +775,7 @@ class portdbapi(dbapi):
try:
if (
threading.current_thread() is threading.main_thread()
-and loop is asyncio._safe_loop()
+and loop._loop is asyncio._safe_loop()._loop
):
# In this case use self._doebuild_settings_lock to manage
concurrency.
deallocate_config = loop.create_future()
diff --git a/lib/portage/tests/ebuild/test_doebuild_spawn.py
b/lib/portage/tests/ebuild/test_doebuild_spawn.py
index 9fb2c7fdd4..cac844f8fb 100644
--- a/lib/portage/tests/ebuild/test_doebuild_spawn.py
+++ b/lib/portage/tests/ebuild/test_doebuild_spawn.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2015 Gentoo Foundation
+# Copyright 2010-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import textwrap
@@ -86,6 +86,7 @@ class DoebuildSpawnTestCase(TestCase):
open(os.path.join(settings["T"], "environment"), "wb").close()
scheduler = SchedulerInterface(global_event_loop())
+self.assertTrue(scheduler._loop is global_event_loop()._loop)
for phase in ("_internal_test",):
# Test EbuildSpawnProcess by calling doebuild.spawn() with
# returnpid=False. This case is no longer used by portage
diff --git a/lib/portage/tests/ebuild/test_ipc_daemon.py
b/lib/portage/tests/ebuild/test_ipc_daemon.py
index 0beb69ddfc..b8777fe94b 100644
--- a/lib/portage/tests/ebuild/test_ipc_daemon.py
+++ b/lib/portage/tests/ebuild/test_ipc_daemon.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2023 Gentoo Authors
+# Copyright 2010-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import tempfile
@@ -77,6 +77,7 @@ class IpcDaemonTestCase(TestCase):
task_scheduler = TaskScheduler(
iter([daemon, proc]), max_jobs=2, event_loop=event_loop
)
+self.assertTrue(task_scheduler._loop is event_loop._loop)
self.received_command = False
diff --git a/lib/portage/util/_async/SchedulerInterface.py
b/lib/portage/util/_async/SchedulerInterface.py
index 43a42adff2..485958491a 100644
--- a/lib/portage/util/_async/SchedulerInterface.py
+++ b/lib/portage/util/_async/SchedulerInterface.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2021 Gentoo Authors
+# Copyright 2012-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import gzip
@@ -49,6 +49,13 @@ class SchedulerInterface(SlotObject):
for k in self._event_loop_attrs:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/, bin/, man/
commit: 749d231d57fb64d3f2e909dfbba26ca7c1ab9d0e
Author: YiFei Zhu gmail com>
AuthorDate: Mon Jan 29 08:50:28 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 21 02:13:04 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=749d231d
estrip: Support debug info deduplication with sys-devel/dwz
This this gates behind FEATURES=dedupdebug, and packages can
opt-out with RESTRICT=dedupdebug, though I'm not sure why a package
would need to opt this out, unless dwz could break something I'm not
aware of...
Bug: https://bugs.gentoo.org/906368
Signed-off-by: YiFei Zhu gmail.com>
Closes: https://github.com/gentoo/portage/pull/1245
Signed-off-by: Sam James gentoo.org>
bin/estrip | 35 +--
lib/_emerge/EbuildPhase.py | 1 +
lib/portage/const.py | 1 +
man/ebuild.5 | 3 +++
man/make.conf.5| 8 +++-
5 files changed, 45 insertions(+), 3 deletions(-)
diff --git a/bin/estrip b/bin/estrip
index 3ac6a16927..2d9d50922a 100755
--- a/bin/estrip
+++ b/bin/estrip
@@ -15,8 +15,8 @@ exp_tf() {
eval ${var}_${flag}=$(tf has ${flag} ${!var})
done
}
-exp_tf FEATURES compressdebug installsources nostrip splitdebug xattr
-exp_tf PORTAGE_RESTRICT binchecks installsources splitdebug strip
+exp_tf FEATURES compressdebug dedupdebug installsources nostrip splitdebug
xattr
+exp_tf PORTAGE_RESTRICT binchecks dedupdebug installsources splitdebug strip
if ! ___eapi_has_prefix_variables; then
EPREFIX= ED=${D}
@@ -201,6 +201,10 @@ fi
[[ ${debugedit} ]] && debugedit_found=true || debugedit_found=false
debugedit_warned=false
+dwz=$(type -P dwz)
+[[ ${dwz} ]] && dwz_found=true || dwz_found=false
+dwz_warned=false
+
__multijob_init
# Setup ${T} filesystem layout that we care about.
@@ -248,6 +252,32 @@ __try_symlink() {
die "failed to create symlink '${name}'"
}
+# Usage: dedup_elf_debug
+dedup_elf_debug() {
+ ${FEATURES_dedupdebug} || return 0
+ ${PORTAGE_RESTRICT_dedupdebug} && return 0
+
+ debug-print-function "${FUNCNAME}" "$@"
+
+ if ! ${dwz_found} ; then
+ if ! ${dwz_warned} ; then
+ dwz_warned=true
+ ewarn "FEATURES=dedupdebug is enabled but the dwz
binary could not be"
+ ewarn "found. This feature will not work unless dwz is
installed!"
+ fi
+ return 0
+ fi
+
+ local src=$1 # File to dedup debug symbols
+ local inode_dedupdebug=$2 # Temp path for hard link tracking
+
+ # We already dedupdebug-ed this inode.
+ [[ -L ${inode_dedupdebug} ]] && return 0
+
+ "${dwz}" -- "${src}"
+ touch "${inode_dedupdebug}"
+}
+
# Usage: save_elf_debug [splitdebug]
save_elf_debug() {
${FEATURES_splitdebug} || return 0
@@ -355,6 +385,7 @@ process_elf() {
xt_data=$(dump_xattrs "${x}")
fi
save_elf_sources "${x}"
+ dedup_elf_debug "${x}" "${inode_link}_dedupdebug"
fi
if ${strip_this} ; then
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index c81bf54a81..3b366f39c7 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -67,6 +67,7 @@ class EbuildPhase(CompositeTask):
_features_display = (
"ccache",
"compressdebug",
+"dedupdebug",
"distcc",
"fakeroot",
"installsources",
diff --git a/lib/portage/const.py b/lib/portage/const.py
index 1f468a1abf..2154213b7b 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -176,6 +176,7 @@ SUPPORTED_FEATURES = frozenset(
"compressdebug",
"compress-index",
"config-protect-if-modified",
+"dedupdebug",
"digest",
"distcc",
"distlocks",
diff --git a/man/ebuild.5 b/man/ebuild.5
index aad189bb00..f849f20a29 100644
--- a/man/ebuild.5
+++ b/man/ebuild.5
@@ -754,6 +754,9 @@ see the \fBQA CONTROL VARIABLES\fR section for more
specific exemptions.
.I bindist
Distribution of built packages is restricted.
.TP
+.I dedupdebug
+Disables dedupdebug for specific packages.
+.TP
.I fetch
like \fImirror\fR but the files will not be fetched via \fBSRC_URI\fR either.
.TP
diff --git a/man/make.conf.5 b/man/make.conf.5
index 20dc9cb52d..23d8408544 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -465,13 +465,19 @@ redundant on\-the\-fly compression. The resulting file
will be called
.B compressdebug
Compress the debug sections in the split debug files with zlib to save
space. See \fBsplitdebug\fR for general split debug
-information (upon which this feature depends).
+information (upon which this feature depends). See also \fBdedupdebug\fR
+for further debug info size reduction.
.TP
.B config\-protect\-if\-modified
This causes the \fBCONFIG_PROTECT\fR behavior to be skipped
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/
commit: 67663e586320347e604727aef006bfe027bcd470
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 12 02:10:52 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 12 13:57:42 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=67663e58
EbuildBuild: Execute EbuildFetchonly in subprocess
Execute EbuildFetchonly in a subprocess since it needs to
run the event loop itself (even for pretend mode since it
may need to fetch mirror layouts as reported in bug 702154).
Also remove obsolete loop.is_running() case in doebuild
related to bug 601252, since it will fail if a mirror
layout file needs to be fetched, so we should just assume
that the event loop is not running.
Set fetched = 2 for listonly (pretend) mode in the fetch
function so that EbuildFetchonly will properly report
success for this mode.
Pass ebuild_path to EbuildFetchonly, and also remove
setcpv call from EbuildFetchonly since it's called earlier
in EbuildBuild.
Bug: https://bugs.gentoo.org/924287
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildBuild.py | 39 +-
lib/_emerge/EbuildFetchonly.py | 10 +++--
lib/portage/package/ebuild/doebuild.py | 30 --
lib/portage/package/ebuild/fetch.py| 1 +
4 files changed, 33 insertions(+), 47 deletions(-)
diff --git a/lib/_emerge/EbuildBuild.py b/lib/_emerge/EbuildBuild.py
index 81cbfdc085..d4a4c6dacb 100644
--- a/lib/_emerge/EbuildBuild.py
+++ b/lib/_emerge/EbuildBuild.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -22,6 +22,7 @@ from portage.package.ebuild.digestcheck import digestcheck
from portage.package.ebuild.doebuild import _check_temp_dir
from portage.package.ebuild._spawn_nofetch import SpawnNofetchWithoutBuilddir
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util.futures.executor.fork import ForkExecutor
from portage.util.path import first_existing
@@ -152,29 +153,25 @@ class EbuildBuild(CompositeTask):
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(
+ebuild_path=self._ebuild_path,
fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings,
)
-retval = fetcher.execute()
-if retval == os.EX_OK:
-self._current_task = None
-self.returncode = os.EX_OK
-self._async_wait()
-else:
-# For pretend mode, the convention it to execute
-# pkg_nofetch and return a successful exitcode.
-self._start_task(
-SpawnNofetchWithoutBuilddir(
-background=self.background,
-
portdb=self.pkg.root_config.trees[self._tree].dbapi,
-ebuild_path=self._ebuild_path,
-scheduler=self.scheduler,
-settings=self.settings,
+# Execute EbuildFetchonly in a subprocess since it needs to
+# run the event loop itself (even for pretend mode since it
+# may need to fetch mirror layouts as reported in bug 702154).
+self._start_task(
+AsyncTaskFuture(
+background=self.background,
+scheduler=self.scheduler,
+future=self.scheduler.run_in_executor(
+ForkExecutor(loop=self.scheduler), fetcher.execute
),
-self._default_final_exit,
-)
+),
+self._fetchonly_exit,
+)
return
quiet_setting = settings.get("PORTAGE_QUIET", False)
@@ -241,8 +238,12 @@ class EbuildBuild(CompositeTask):
self._start_task(pre_clean_phase, self._pre_clean_exit)
def _fetchonly_exit(self, fetcher):
+if not fetcher.cancelled and isinstance(fetcher, AsyncTaskFuture):
+# Set returncode from EbuildFetchonly.execute() result, since
+# it can fail if it can't resolve a mirror for a file.
+fetcher.returncode = fetcher.future.result()
self._final_exit(fetcher)
-if self.returncode != os.EX_OK:
+if not self.cancelled and self.returncode != os.EX_OK:
self.returncode = None
portdb = self.pkg.root_config.trees[self._tree].dbapi
self._start_task(
diff --git a/lib/_emerge/EbuildFetchonly.py b/lib/_emerge/EbuildFetchonly.py
index e887dd858d..c806122bea 100644
--- a/lib/_emerge/EbuildFetchonly.py
+++
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/
commit: a69c1b853a47346192950c91b088163490287350 Author: Zac Medico gentoo org> AuthorDate: Sat Feb 3 21:27:45 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 7 00:49:26 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a69c1b85 process.spawn: Use multiprocessing.Process for returnproc Use multiprocessing.Process for returnproc, so that fork will stop being used when python makes "spawn" the default multiprocessing start method. Continue to use _start_fork when returnproc is not enabled, for backward compatibility. Ultimately, it can be removed at the same time as the returnpid parameter. The _setup_pipes_after_fork wrapper prevents a "Bad file descriptor" error by making fd_pipes inheritable on exec for bug 923755. ForkProcess does not handle this because its target function does not necessarily exec. Bug: https://bugs.gentoo.org/916566 Bug: https://bugs.gentoo.org/923755 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/SpawnProcess.py | 4 +- lib/portage/process.py | 92 ++--- 2 files changed, 82 insertions(+), 14 deletions(-) diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py index 716e94d665..b63afae01c 100644 --- a/lib/_emerge/SpawnProcess.py +++ b/lib/_emerge/SpawnProcess.py @@ -241,7 +241,9 @@ class SpawnProcess(SubProcess): got_pty, master_fd, slave_fd = _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) -def _spawn(self, args: list[str], **kwargs) -> portage.process.Process: +def _spawn( +self, args: list[str], **kwargs +) -> portage.process.MultiprocessingProcess: spawn_func = portage.process.spawn if self._selinux_type is not None: diff --git a/lib/portage/process.py b/lib/portage/process.py index b223ecb887..20327b38bc 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -19,7 +19,7 @@ import warnings from dataclasses import dataclass from functools import lru_cache -from typing import Any, Optional, Callable +from typing import Any, Optional, Callable, Union from portage import os from portage import _encodings @@ -28,6 +28,7 @@ import portage portage.proxy.lazyimport.lazyimport( globals(), +"portage.util._async.ForkProcess:ForkProcess", "portage.util._eventloop.global_event_loop:global_event_loop", "portage.util.futures:asyncio", "portage.util:dump_traceback,writemsg,writemsg_level", @@ -296,12 +297,19 @@ class AbstractProcess: class Process(AbstractProcess): """ -An object that wraps OS processes created by spawn. -In the future, spawn will return objects of a different type -but with a compatible interface to this class, in order -to encapsulate implementation-dependent objects like -multiprocessing.Process which are designed to manage -the process lifecycle and need to persist until it exits. +An object that wraps OS processes which do not have an +associated multiprocessing.Process instance. Ultimately, +we need to stop using os.fork() to create these processes +because it is unsafe for threaded processes as discussed +in https://github.com/python/cpython/issues/84559. + +Note that if subprocess.Popen is used without pass_fds +or preexec_fn parameters, then it avoids using os.fork() +by instead using posix_spawn. This approach is not used +by spawn because it needs to execute python code prior +to exec, so it instead uses multiprocessing.Process, +which only uses os.fork() when the multiprocessing start +method is fork. """ def __init__(self, pid: int): @@ -461,7 +469,7 @@ def spawn( unshare_mount=False, unshare_pid=False, warn_on_large_env=False, -): +) -> Union[int, MultiprocessingProcess, list[int]]: """ Spawns a given command. @@ -479,8 +487,8 @@ def spawn( @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean -@param returnproc: Return a Process object for a successful spawn (conflicts with logfile parameter). -NOTE: This requires the caller to asynchronously wait for the Process. +@param returnproc: Return a MultiprocessingProcess instance (conflicts with logfile parameter). +NOTE: This requires the caller to asynchronously wait for the MultiprocessingProcess instance. @type returnproc: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @@ -626,7 +634,9 @@ def spawn( # fork, so that the result is cached in the main process. bool(groups) -pid = _start_fork( +start_func = _start_proc if returnproc else _start_fork + +pid = start_func( _exec_wrapper, args=( binary, @@ -652,6 +662,10 @@ def spawn( close_fds=close_fds, ) +
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/
commit: 190e8ad1a238d20d782235dd1faa6b00d1b3fd4a
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 3 21:24:49 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 7 00:55:45 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=190e8ad1
doebuild: Add returnproc parameter and deprecate returnpid
Raise NotImplementedError if returnproc is enabled for anything
other than the "depend" phase, since corresponding returnpid
support has long been deprecated.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py | 4 +--
lib/portage/package/ebuild/doebuild.py | 47 ++
2 files changed, 39 insertions(+), 12 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index a7c9650d74..f4f685e81c 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -124,7 +124,7 @@ class EbuildMetadataPhase(SubProcess):
mydbapi=self.portdb,
tree="porttree",
fd_pipes=fd_pipes,
-returnpid=True,
+returnproc=True,
)
settings.pop("PORTAGE_PIPE_FD", None)
@@ -137,7 +137,7 @@ class EbuildMetadataPhase(SubProcess):
self._async_wait()
return
-self._proc = portage.process.Process(retval[0])
+self._proc = retval
def _output_handler(self):
while True:
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index e10b884e08..1c89af5ac8 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -19,6 +19,7 @@ import sys
import tempfile
from textwrap import wrap
import time
+from typing import Union
import warnings
import zlib
@@ -246,14 +247,21 @@ def _doebuild_spawn(phase, settings, actionmap=None,
**kwargs):
def _spawn_phase(
-phase, settings, actionmap=None, returnpid=False, logfile=None, **kwargs
+phase,
+settings,
+actionmap=None,
+returnpid=False,
+returnproc=False,
+logfile=None,
+**kwargs,
):
-if returnpid:
+if returnproc or returnpid:
return _doebuild_spawn(
phase,
settings,
actionmap=actionmap,
returnpid=returnpid,
+returnproc=returnproc,
logfile=logfile,
**kwargs,
)
@@ -725,7 +733,8 @@ def doebuild(
prev_mtimes=None,
fd_pipes=None,
returnpid=False,
-):
+returnproc=False,
+) -> Union[int, portage.process.MultiprocessingProcess, list[int]]:
"""
Wrapper function that invokes specific ebuild phases through the spawning
of ebuild.sh
@@ -762,9 +771,15 @@ def doebuild(
for example.
@type fd_pipes: Dictionary
@param returnpid: Return a list of process IDs for a successful spawn, or
-an integer value if spawn is unsuccessful. NOTE: This requires the
-caller clean up all returned PIDs.
+an integer value if spawn is unsuccessful. This parameter is
supported
+supported only when mydo is "depend". NOTE: This requires the
caller clean
+up all returned PIDs.
@type returnpid: Boolean
+@param returnproc: Return a MultiprocessingProcess instance for a
successful spawn, or
+an integer value if spawn is unsuccessful. This parameter is
supported
+supported only when mydo is "depend". NOTE: This requires the
caller to
+asynchronously wait for the MultiprocessingProcess instance.
+@type returnproc: Boolean
@rtype: Boolean
@return:
1. 0 for success
@@ -867,17 +882,25 @@ def doebuild(
writemsg("\n", noiselevel=-1)
return 1
-if returnpid and mydo != "depend":
+if (returnproc or returnpid) and mydo != "depend":
# This case is not supported, since it bypasses the EbuildPhase class
# which implements important functionality (including post phase hooks
# and IPC for things like best/has_version and die).
+if returnproc:
+raise NotImplementedError(f"returnproc not implemented for phase
{mydo}")
warnings.warn(
"portage.doebuild() called "
"with returnpid parameter enabled. This usage will "
"not be supported in the future.",
-DeprecationWarning,
+UserWarning,
stacklevel=2,
)
+elif returnpid:
+warnings.warn(
+"The portage.doebuild() returnpid parameter is deprecated and
replaced by returnproc",
+UserWarning,
+stacklevel=1,
+)
if mydo == "fetchall":
fetchall = 1
@@ -1027,10 +1050,14 @@ def doebuild(
# get possible slot information from the deps file
if mydo == "depend":
-if not returnpid:
-raise
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
commit: e8b31c86eaed645a8740fb2844e2935aee161e43
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 5 05:55:11 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 6 01:30:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e8b31c86
ForkProcess: Prevent redundant pipe and set_term_size recursion
When process.spawn is updated to call ForkProcess
for bug 916566, it needs to avoid recursion via
set_term_size.
Bug: https://bugs.gentoo.org/916566
Bug: https://bugs.gentoo.org/923750
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/SpawnProcess.py| 31 ---
lib/portage/util/_async/ForkProcess.py | 28 +++-
2 files changed, 47 insertions(+), 12 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 7f4a23892b..716e94d665 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -41,7 +41,7 @@ class SpawnProcess(SubProcess):
)
__slots__ = (
-("args", "log_filter_file")
+("args", "create_pipe", "log_filter_file")
+ _spawn_kwarg_names
+ (
"_main_task",
@@ -60,15 +60,30 @@ class SpawnProcess(SubProcess):
else:
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
+log_file_path = None
if fd_pipes or self.logfile or not self.background:
-master_fd, slave_fd = self._pipe(fd_pipes)
+if self.create_pipe is not False:
+master_fd, slave_fd = self._pipe(fd_pipes)
-can_log = self._can_log(slave_fd)
-if can_log:
-log_file_path = self.logfile
+can_log = self._can_log(slave_fd)
+if can_log:
+log_file_path = self.logfile
else:
-log_file_path = None
+if self.logfile:
+raise NotImplementedError(
+"logfile conflicts with create_pipe=False"
+)
+# When called via process.spawn and ForkProcess._start,
+# SpawnProcess will have created a pipe earlier, so it
+# would be redundant to do it here (it could also trigger
+# spawn recursion via set_term_size as in bug 923750).
+# Use /dev/null for master_fd, triggering early return
+# of _main, followed by _async_waitpid.
+# TODO: Optimize away the need for master_fd here.
+master_fd = os.open(os.devnull, os.O_RDONLY)
+slave_fd = None
+can_log = False
null_input = None
if not self.background or 0 in fd_pipes:
@@ -97,7 +112,9 @@ class SpawnProcess(SubProcess):
fd_pipes_orig = fd_pipes.copy()
-if log_file_path is not None or self.background:
+if slave_fd is None:
+pass
+elif log_file_path is not None or self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index 3acbe34fc6..cb240d0712 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -75,12 +75,29 @@ class ForkProcess(SpawnProcess):
self.fd_pipes.setdefault(0, portage._get_stdin().fileno())
self.fd_pipes.setdefault(1, sys.__stdout__.fileno())
self.fd_pipes.setdefault(2, sys.__stderr__.fileno())
-stdout_fd = os.dup(self.fd_pipes[1])
+if self.create_pipe is not False:
+stdout_fd = os.dup(self.fd_pipes[1])
if self._HAVE_SEND_HANDLE:
-master_fd, slave_fd = self._pipe(self.fd_pipes)
-self.fd_pipes[1] = slave_fd
-self.fd_pipes[2] = slave_fd
+if self.create_pipe is not False:
+master_fd, slave_fd = self._pipe(self.fd_pipes)
+self.fd_pipes[1] = slave_fd
+self.fd_pipes[2] = slave_fd
+else:
+if self.logfile:
+raise NotImplementedError(
+"logfile conflicts with create_pipe=False"
+)
+# When called via process.spawn, SpawnProcess
+# will have created a pipe earlier, so it would be
+# redundant to do it here (it could also trigger spawn
+# recursion via set_term_size as in bug 923750). Use
+# /dev/null for master_fd, triggering early return
+# of _main, followed by _async_waitpid.
+# TODO: Optimize away the need for master_fd here.
+master_fd = os.open(os.devnull, os.O_RDONLY)
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/
commit: 0e4378fd18bcefde4472c7ae65e99578cf47dce6
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 4 07:59:26 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 4 08:00:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0e4378fd
Revert "doebuild: Add returnproc parameter and deprecate returnpid"
This reverts commit 6aeee45d9b88d7dc8c450ba86975e4b017db203c.
It triggered a "Bad file descriptor" during the instprep
phase as reported in bug 923755.
Bug: https://bugs.gentoo.org/923755
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py | 5 ++--
lib/portage/package/ebuild/doebuild.py | 47 --
2 files changed, 12 insertions(+), 40 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index b8164f2b16..a7c9650d74 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -108,7 +108,6 @@ class EbuildMetadataPhase(SubProcess):
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
)
-os.set_inheritable(slave_fd, True)
fd_pipes[slave_fd] = slave_fd
settings["PORTAGE_PIPE_FD"] = str(slave_fd)
@@ -125,7 +124,7 @@ class EbuildMetadataPhase(SubProcess):
mydbapi=self.portdb,
tree="porttree",
fd_pipes=fd_pipes,
-returnproc=True,
+returnpid=True,
)
settings.pop("PORTAGE_PIPE_FD", None)
@@ -138,7 +137,7 @@ class EbuildMetadataPhase(SubProcess):
self._async_wait()
return
-self._proc = retval
+self._proc = portage.process.Process(retval[0])
def _output_handler(self):
while True:
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 59a2b86a38..e10b884e08 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -19,7 +19,6 @@ import sys
import tempfile
from textwrap import wrap
import time
-from typing import Union
import warnings
import zlib
@@ -247,21 +246,14 @@ def _doebuild_spawn(phase, settings, actionmap=None,
**kwargs):
def _spawn_phase(
-phase,
-settings,
-actionmap=None,
-returnpid=False,
-returnproc=False,
-logfile=None,
-**kwargs,
+phase, settings, actionmap=None, returnpid=False, logfile=None, **kwargs
):
-if returnproc or returnpid:
+if returnpid:
return _doebuild_spawn(
phase,
settings,
actionmap=actionmap,
returnpid=returnpid,
-returnproc=returnproc,
logfile=logfile,
**kwargs,
)
@@ -733,8 +725,7 @@ def doebuild(
prev_mtimes=None,
fd_pipes=None,
returnpid=False,
-returnproc=False,
-) -> Union[int, portage.process.MultiprocessingProcess, list[int]]:
+):
"""
Wrapper function that invokes specific ebuild phases through the spawning
of ebuild.sh
@@ -771,15 +762,9 @@ def doebuild(
for example.
@type fd_pipes: Dictionary
@param returnpid: Return a list of process IDs for a successful spawn, or
-an integer value if spawn is unsuccessful. This parameter is
supported
-supported only when mydo is "depend". NOTE: This requires the
caller clean
-up all returned PIDs.
+an integer value if spawn is unsuccessful. NOTE: This requires the
+caller clean up all returned PIDs.
@type returnpid: Boolean
-@param returnproc: Return a MultiprocessingProcess instance for a
successful spawn, or
-an integer value if spawn is unsuccessful. This parameter is
supported
-supported only when mydo is "depend". NOTE: This requires the
caller to
-asynchronously wait for the MultiprocessingProcess instance.
-@type returnproc: Boolean
@rtype: Boolean
@return:
1. 0 for success
@@ -882,25 +867,17 @@ def doebuild(
writemsg("\n", noiselevel=-1)
return 1
-if (returnproc or returnpid) and mydo != "depend":
+if returnpid and mydo != "depend":
# This case is not supported, since it bypasses the EbuildPhase class
# which implements important functionality (including post phase hooks
# and IPC for things like best/has_version and die).
-if returnproc:
-raise NotImplementedError(f"returnproc not implemented for phase
{mydo}")
warnings.warn(
"portage.doebuild() called "
"with returnpid parameter enabled. This usage will "
"not be supported in the future.",
-UserWarning,
+DeprecationWarning,
stacklevel=2,
)
-elif returnpid:
-warnings.warn(
-"The portage.doebuild() returnpid paramenter is deprecated and
replaced by returnproc",
-
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/
commit: 6aeee45d9b88d7dc8c450ba86975e4b017db203c
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 3 21:24:49 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 3 22:40:34 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6aeee45d
doebuild: Add returnproc parameter and deprecate returnpid
Raise NotImplementedError if returnproc is enabled for anything
other than the "depend" phase, since corresponding returnpid
support has long been deprecated.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py | 5 ++--
lib/portage/package/ebuild/doebuild.py | 47 ++
2 files changed, 40 insertions(+), 12 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index a7c9650d74..b8164f2b16 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -108,6 +108,7 @@ class EbuildMetadataPhase(SubProcess):
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
)
+os.set_inheritable(slave_fd, True)
fd_pipes[slave_fd] = slave_fd
settings["PORTAGE_PIPE_FD"] = str(slave_fd)
@@ -124,7 +125,7 @@ class EbuildMetadataPhase(SubProcess):
mydbapi=self.portdb,
tree="porttree",
fd_pipes=fd_pipes,
-returnpid=True,
+returnproc=True,
)
settings.pop("PORTAGE_PIPE_FD", None)
@@ -137,7 +138,7 @@ class EbuildMetadataPhase(SubProcess):
self._async_wait()
return
-self._proc = portage.process.Process(retval[0])
+self._proc = retval
def _output_handler(self):
while True:
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index e10b884e08..59a2b86a38 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -19,6 +19,7 @@ import sys
import tempfile
from textwrap import wrap
import time
+from typing import Union
import warnings
import zlib
@@ -246,14 +247,21 @@ def _doebuild_spawn(phase, settings, actionmap=None,
**kwargs):
def _spawn_phase(
-phase, settings, actionmap=None, returnpid=False, logfile=None, **kwargs
+phase,
+settings,
+actionmap=None,
+returnpid=False,
+returnproc=False,
+logfile=None,
+**kwargs,
):
-if returnpid:
+if returnproc or returnpid:
return _doebuild_spawn(
phase,
settings,
actionmap=actionmap,
returnpid=returnpid,
+returnproc=returnproc,
logfile=logfile,
**kwargs,
)
@@ -725,7 +733,8 @@ def doebuild(
prev_mtimes=None,
fd_pipes=None,
returnpid=False,
-):
+returnproc=False,
+) -> Union[int, portage.process.MultiprocessingProcess, list[int]]:
"""
Wrapper function that invokes specific ebuild phases through the spawning
of ebuild.sh
@@ -762,9 +771,15 @@ def doebuild(
for example.
@type fd_pipes: Dictionary
@param returnpid: Return a list of process IDs for a successful spawn, or
-an integer value if spawn is unsuccessful. NOTE: This requires the
-caller clean up all returned PIDs.
+an integer value if spawn is unsuccessful. This parameter is
supported
+supported only when mydo is "depend". NOTE: This requires the
caller clean
+up all returned PIDs.
@type returnpid: Boolean
+@param returnproc: Return a MultiprocessingProcess instance for a
successful spawn, or
+an integer value if spawn is unsuccessful. This parameter is
supported
+supported only when mydo is "depend". NOTE: This requires the
caller to
+asynchronously wait for the MultiprocessingProcess instance.
+@type returnproc: Boolean
@rtype: Boolean
@return:
1. 0 for success
@@ -867,17 +882,25 @@ def doebuild(
writemsg("\n", noiselevel=-1)
return 1
-if returnpid and mydo != "depend":
+if (returnproc or returnpid) and mydo != "depend":
# This case is not supported, since it bypasses the EbuildPhase class
# which implements important functionality (including post phase hooks
# and IPC for things like best/has_version and die).
+if returnproc:
+raise NotImplementedError(f"returnproc not implemented for phase
{mydo}")
warnings.warn(
"portage.doebuild() called "
"with returnpid parameter enabled. This usage will "
"not be supported in the future.",
-DeprecationWarning,
+UserWarning,
stacklevel=2,
)
+elif returnpid:
+warnings.warn(
+"The portage.doebuild() returnpid paramenter is deprecated and
replaced by returnproc",
+UserWarning,
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/, lib/portage/tests/ebuild/
commit: 305612d1b04aa06d3d1a1c8b51d046a644742fd5 Author: Zac Medico gentoo org> AuthorDate: Sat Feb 3 21:27:45 2024 + Commit: Zac Medico gentoo org> CommitDate: Sat Feb 3 21:27:45 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=305612d1 process.spawn: Use multiprocessing.Process for returnproc Use multiprocessing.Process for returnproc, so that fork will stop being used when python makes "spawn" the default multiprocessing start method. Continue to use _start_fork when returnproc is not enabled, for backward compatibility. Ultimately, it can be removed at the same time as the returnpid parameter. Bug: https://bugs.gentoo.org/916566 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/SpawnProcess.py| 4 +- lib/portage/process.py | 72 ++ lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 6 +- 3 files changed, 67 insertions(+), 15 deletions(-) diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py index 7f4a23892b..5e582e3322 100644 --- a/lib/_emerge/SpawnProcess.py +++ b/lib/_emerge/SpawnProcess.py @@ -224,7 +224,9 @@ class SpawnProcess(SubProcess): got_pty, master_fd, slave_fd = _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) -def _spawn(self, args: list[str], **kwargs) -> portage.process.Process: +def _spawn( +self, args: list[str], **kwargs +) -> portage.process.MultiprocessingProcess: spawn_func = portage.process.spawn if self._selinux_type is not None: diff --git a/lib/portage/process.py b/lib/portage/process.py index 01426179d7..d64ffa924f 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -19,7 +19,7 @@ import warnings from dataclasses import dataclass from functools import lru_cache -from typing import Any, Optional, Callable +from typing import Any, Optional, Callable, Union from portage import os from portage import _encodings @@ -28,6 +28,7 @@ import portage portage.proxy.lazyimport.lazyimport( globals(), +"portage.util._async.ForkProcess:ForkProcess", "portage.util._eventloop.global_event_loop:global_event_loop", "portage.util.futures:asyncio", "portage.util:dump_traceback,writemsg,writemsg_level", @@ -296,12 +297,19 @@ class AbstractProcess: class Process(AbstractProcess): """ -An object that wraps OS processes created by spawn. -In the future, spawn will return objects of a different type -but with a compatible interface to this class, in order -to encapsulate implementation-dependent objects like -multiprocessing.Process which are designed to manage -the process lifecycle and need to persist until it exits. +An object that wraps OS processes which do not have an +associated multiprocessing.Process instance. Ultimately, +we need to stop using os.fork() to create these processes +because it is unsafe for threaded processes as discussed +in https://github.com/python/cpython/issues/84559. + +Note that if subprocess.Popen is used without pass_fds +or preexec_fn parameters, then it avoids using os.fork() +by instead using posix_spawn. This approach is not used +by spawn because it needs to execute python code prior +to exec, so it instead uses multiprocessing.Process, +which only uses os.fork() when the multiprocessing start +method is fork. """ def __init__(self, pid: int): @@ -461,7 +469,7 @@ def spawn( unshare_mount=False, unshare_pid=False, warn_on_large_env=False, -): +) -> Union[int, MultiprocessingProcess, list[int]]: """ Spawns a given command. @@ -479,8 +487,8 @@ def spawn( @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean -@param returnproc: Return a Process object for a successful spawn (conflicts with logfile parameter). -NOTE: This requires the caller to asynchronously wait for the Process. +@param returnproc: Return a MultiprocessingProcess instance (conflicts with logfile parameter). +NOTE: This requires the caller to asynchronously wait for the MultiprocessingProcess instance. @type returnproc: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @@ -623,7 +631,9 @@ def spawn( # fork, so that the result is cached in the main process. bool(groups) -pid = _start_fork( +start_func = _start_proc if returnproc else _start_fork + +pid = start_func( _exec_wrapper, args=( binary, @@ -649,6 +659,10 @@ def spawn( close_fds=close_fds, ) +if returnproc: +# _start_proc returns a MultiprocessingProcess instance. +return pid + if not isinstance(pid, int): raise
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: bf3d091de8702f0c95e5530d03c6e925008ee80a
Author: Zac Medico gentoo org>
AuthorDate: Sun Dec 24 05:12:55 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Dec 24 19:29:25 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bf3d091d
depclean: drop direct circular deps in merge order calculation
Drop direct circular deps in the depclean merge order calculation,
since it does not handle them well as shown by the test case
for bug 916135.
Bug: https://bugs.gentoo.org/916135
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/actions.py| 8 ++--
lib/portage/tests/resolver/test_depclean_order.py | 10 ++
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 86ba7f77a5..13bb75931c 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import collections
@@ -1642,7 +1642,11 @@ def _calc_depclean(settings, trees, ldpath_mtimes,
myopts, action, args_set, spi
if mypriority.runtime:
mypriority.runtime_slot_op = True
-graph.add(child_node, node, priority=mypriority)
+# Drop direct circular deps because the unmerge
order
+# calculation does not handle them well as
demonstrated
+# by the test case for bug 916135.
+if child_node is not node:
+graph.add(child_node, node,
priority=mypriority)
if debug:
writemsg_level("\nunmerge digraph:\n\n", noiselevel=-1,
level=logging.DEBUG)
diff --git a/lib/portage/tests/resolver/test_depclean_order.py
b/lib/portage/tests/resolver/test_depclean_order.py
index 867b1a54ca..23b5e755c3 100644
--- a/lib/portage/tests/resolver/test_depclean_order.py
+++ b/lib/portage/tests/resolver/test_depclean_order.py
@@ -1,8 +1,6 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import pytest
-
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
@@ -60,8 +58,12 @@ class SimpleDepcleanTestCase(TestCase):
finally:
playground.cleanup()
-@pytest.mark.xfail()
def testIDEPENDDepclean(self):
+"""
+Test for bug 916135, where a direct circular dependency caused
+the unmerge order to fail to account for IDEPEND.
+"""
+
ebuilds = {
"dev-util/A-1": {},
"dev-libs/B-1": {
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/emerge/, lib/portage/tests/dep/, ...
commit: af550b8b5cb91f27b26d6800c3b4cdd2d86a46e6
Author: Sam James gentoo org>
AuthorDate: Mon Sep 4 15:38:18 2023 +
Commit: Sam James gentoo org>
CommitDate: Sun Dec 10 22:01:48 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=af550b8b
ebuild: inject implicit libc RDEPEND
Inject >=${LIBC_PROVIDER}-${VERSION_OF_LIBC_PROVIDER} into RDEPEND.
We already try to upgrade the virtual/libc provider and its deps aggressively
but that's not so helpful if there's a binpkg for, say, one of its deps
available
built against a newer glibc - which ends up breaking the system because our
glibc isn't new enough (symbol versioning).
This is a long-standing problem for binpkg users and one of the last
showstoppers
for being able to use them reliably.
Note that this applies to source installs too, although it matters less there;
it'll make downgrading libc a bit harder for people who want to do that, but
users
are already prevented from doing that (pkg_* check) for glibc, and I don't
really
see it as a bad thing to effectively propagate this to other libcs.
To fully solve the problem, we should arguably do at least one of the following:
1) Implicit >= on anything (not just libc) which installs ELF (at least a
PROVIDES);
2) Implement the suggestion in bug #753500 based on analysis of used versioned
symbols (*).
But libc is really the critical one and the one where things explode pretty
badly,
especially combined with us trying to Do The Right Thing for non-binpkg cases
(aggressively upgrading libc before anything else). The other cases don't matter
so much as they get upgraded later and by that point, the library is usually
done.
(It's not really clear if downgrading musl works and even if it is supported,
I'm not really sure it's a supported case at all, so I'm not bothering to carve
out an exception here. It'd make this far less elegant and I don't see any
benefit
to doing so.)
(*) util-linux is one of the examples I have in mind here as justification
for either point.
Bug: https://bugs.gentoo.org/753500
Bug: https://bugs.gentoo.org/913628
Signed-off-by: Sam James gentoo.org>
lib/_emerge/actions.py | 7 +-
lib/_emerge/depgraph.py | 22 +
lib/portage/dep/__init__.py | 23 +-
lib/portage/dep/libc.py | 83
lib/portage/dep/meson.build | 1 +
lib/portage/package/ebuild/doebuild.py | 56 ++-
lib/portage/tests/dep/meson.build| 1 +
lib/portage/tests/dep/test_libc.py | 81
lib/portage/tests/emerge/meson.build | 1 +
lib/portage/tests/emerge/test_actions.py | 23 +-
lib/portage/tests/emerge/test_libc_dep_inject.py | 551 +++
11 files changed, 831 insertions(+), 18 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index dbd9707a82..ae8796531e 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -41,6 +41,7 @@ from portage.dbapi._expand_new_virt import expand_new_virt
from portage.dbapi.IndexedPortdb import IndexedPortdb
from portage.dbapi.IndexedVardb import IndexedVardb
from portage.dep import Atom, _repo_separator, _slot_separator
+from portage.dep.libc import find_libc_deps
from portage.exception import (
InvalidAtom,
InvalidData,
@@ -2786,10 +2787,8 @@ def relative_profile_path(portdir, abs_profile):
def get_libc_version(vardb: portage.dbapi.vartree.vardbapi) -> list[str]:
libcver = []
-libclist = set()
-for atom in expand_new_virt(vardb, portage.const.LIBC_PACKAGE_ATOM):
-if not atom.blocker:
-libclist.update(vardb.match(atom))
+libclist = find_libc_deps(vardb, True)
+
if libclist:
for cpv in sorted(libclist):
libc_split = portage.catpkgsplit(cpv)[1:]
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 59c78c7354..4612ac2049 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -36,6 +36,7 @@ from portage.dep import (
match_from_list,
_repo_separator,
)
+from portage.dep.libc import find_libc_deps, strip_libc_deps
from portage.dep._slot_operator import ignore_built_slot_operator_deps,
strip_slots
from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use,
_get_eapi_attrs
from portage.exception import (
@@ -2984,6 +2985,19 @@ class depgraph:
else:
depvars = Package._runtime_keys
+eroot = pkg.root_config.settings["EROOT"]
+try:
+libc_deps = self._frozen_config._libc_deps_cache[eroot]
+except (AttributeError, KeyError) as e:
+if isinstance(e, AttributeError):
+self._frozen_config._libc_deps_cache = {}
+
+self._frozen_config._libc_deps_cache[eroot] = find_libc_deps(
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 1d856747ada48f8d32c033091b1156cc655efed3
Author: Zac Medico gentoo org>
AuthorDate: Wed Dec 6 06:05:46 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Dec 6 20:23:14 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1d856747
DepPriority{Normal,Satisfied}Range: weaken _ignore_runtime for cross root
When the dependency parent is for a cross root (ROOT != /) package,
weaken _ignore_runtime in order to tolerate runtime cycles that are
less problematic for cross root packages.
The included test case fails with this error without the fix:
* Error: circular dependencies:
(dev-libs/gmp-6.3.0:0/10.4::test_repo, binary scheduled for merge to
'/tmp/tmp25nwdjn7/cross_root/') depends on
(sys-devel/gcc-13.2.1_p20230826:0/0::test_repo, binary scheduled for merge to
'/tmp/tmp25nwdjn7/cross_root/') (runtime)
(dev-libs/gmp-6.3.0:0/10.4::test_repo, binary scheduled for merge to
'/tmp/tmp25nwdjn7/cross_root/') (runtime_slot_op)
It might be possible to break this cycle
by applying the following change:
- dev-libs/gmp-6.3.0 (Change USE: -cxx)
Bug: https://bugs.gentoo.org/919174
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/DepPriority.py | 4 +-
lib/_emerge/DepPriorityNormalRange.py | 4 +-
lib/_emerge/DepPrioritySatisfiedRange.py | 1 +
lib/_emerge/UnmergeDepPriority.py | 3 +-
lib/_emerge/depgraph.py| 46 --
lib/portage/tests/resolver/meson.build | 1 +
.../tests/resolver/test_cross_dep_priority.py | 164 +
7 files changed, 208 insertions(+), 15 deletions(-)
diff --git a/lib/_emerge/DepPriority.py b/lib/_emerge/DepPriority.py
index 99d38477e2..8d282b937a 100644
--- a/lib/_emerge/DepPriority.py
+++ b/lib/_emerge/DepPriority.py
@@ -1,11 +1,11 @@
-# Copyright 1999-2013 Gentoo Foundation
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.AbstractDepPriority import AbstractDepPriority
class DepPriority(AbstractDepPriority):
-__slots__ = ("satisfied", "optional", "ignored")
+__slots__ = ("cross", "ignored", "optional", "satisfied")
def __int__(self):
"""
diff --git a/lib/_emerge/DepPriorityNormalRange.py
b/lib/_emerge/DepPriorityNormalRange.py
index d7e4381b47..cb0e6c26b1 100644
--- a/lib/_emerge/DepPriorityNormalRange.py
+++ b/lib/_emerge/DepPriorityNormalRange.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2011 Gentoo Foundation
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.DepPriority import DepPriority
@@ -41,7 +41,7 @@ class DepPriorityNormalRange:
# to adjust this appropriately. But only build time dependencies
# are optional right now, so it's not an issue as-is.
return bool(
-not priority.runtime_slot_op
+not (priority.runtime_slot_op and not priority.cross)
and (priority.optional or not priority.buildtime)
)
diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py
b/lib/_emerge/DepPrioritySatisfiedRange.py
index 0d42e7613d..b3bc90c2ff 100644
--- a/lib/_emerge/DepPrioritySatisfiedRange.py
+++ b/lib/_emerge/DepPrioritySatisfiedRange.py
@@ -96,6 +96,7 @@ class DepPrioritySatisfiedRange:
(
(not priority.runtime_slot_op)
or (priority.satisfied and priority.runtime_slot_op)
+or priority.cross
)
and (priority.satisfied or priority.optional or not
priority.buildtime)
)
diff --git a/lib/_emerge/UnmergeDepPriority.py
b/lib/_emerge/UnmergeDepPriority.py
index ff81eff46f..d818bad1b8 100644
--- a/lib/_emerge/UnmergeDepPriority.py
+++ b/lib/_emerge/UnmergeDepPriority.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2013 Gentoo Foundation
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.AbstractDepPriority import AbstractDepPriority
@@ -6,6 +6,7 @@ from _emerge.AbstractDepPriority import AbstractDepPriority
class UnmergeDepPriority(AbstractDepPriority):
__slots__ = (
+"cross",
"ignored",
"optional",
"satisfied",
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 9b09701021..59c78c7354 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -3630,7 +3630,7 @@ class depgraph:
blocker=False,
depth=depth,
parent=pkg,
-priority=self._priority(runtime=True),
+priority=self._priority(cross=self._cross(pkg.root),
runtime=True),
root=pkg.root,
)
if not self._add_dep(dep, allow_unsatisfied=allow_unsatisfied):
@@ -3968,17 +3968,26 @@ class depgraph:
# _dep_disjunctive_stack first, so
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 3487594cd8f46a5c83caaab3a9425321443e5efc
Author: Zac Medico gentoo org>
AuthorDate: Tue Nov 28 04:58:07 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Nov 28 22:41:45 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3487594c
Increase ignore_priority during topological sort for runtime cycle
Fix AlternativesGzipTestCase by increasing ignore_priority in
order to find smaller groups of leaf nodes during topological
sort for runtime cycles. This causes some changes in merge
order for MergeOrderTestCase, but they appear to be acceptable
since they prevent temporarily unsatisfied RDEPEND by relying
on satisfied installed build-time dependencies.
Bug: https://bugs.gentoo.org/917259
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py| 27 --
.../tests/resolver/test_alternatives_gzip.py | 8 +++
lib/portage/tests/resolver/test_merge_order.py | 20
.../tests/resolver/test_rebuild_ghostscript.py | 2 +-
4 files changed, 35 insertions(+), 22 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 29eadba3d1..da37f980ad 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -9478,17 +9478,30 @@ class depgraph:
)
selected_nodes = []
while cycle_digraph:
+# Increase ignore_priority in order to find
+# smaller groups of leaf nodes. This solves
+# bug 917259 which happened because too many
+# leaves were selected at once.
+smallest_leaves = None
for ignore_priority in ignore_priorities:
leaves = cycle_digraph.leaf_nodes(
ignore_priority=ignore_priority
)
-if leaves:
-cycle_digraph.difference_update(leaves)
-selected_nodes.extend(leaves)
-break
-else:
-selected_nodes.extend(cycle_digraph)
-break
+if leaves and (
+smallest_leaves is None
+or len(leaves) < len(smallest_leaves)
+):
+smallest_leaves = leaves
+if len(smallest_leaves) == 1:
+break
+
+if smallest_leaves is None:
+smallest_leaves = [cycle_digraph.order[-1]]
+
+# Only harvest one node at a time, in order to
+# minimize the number of ignored dependencies.
+cycle_digraph.remove(smallest_leaves[0])
+selected_nodes.append(smallest_leaves[0])
if not selected_nodes and myblocker_uninstalls:
# An Uninstall task needs to be executed in order to
diff --git a/lib/portage/tests/resolver/test_alternatives_gzip.py
b/lib/portage/tests/resolver/test_alternatives_gzip.py
index 602ed1756f..e763e84640 100644
--- a/lib/portage/tests/resolver/test_alternatives_gzip.py
+++ b/lib/portage/tests/resolver/test_alternatives_gzip.py
@@ -1,8 +1,6 @@
# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import pytest
-
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
@@ -10,7 +8,6 @@ from portage.tests.resolver.ResolverPlayground import (
)
-@pytest.mark.xfail()
class AlternativesGzipTestCase(TestCase):
def testAlternativesGzip(self):
"""
@@ -19,8 +16,9 @@ class AlternativesGzipTestCase(TestCase):
find_smallest_cycle selects a large cycle and the topological
sort produces poor results when leaf_nodes returns
app-alternatives/gzip as part of a large group of nodes.
-This problem might be solved by implementing a finer-grained
-ignore_priority for leaf_nodes calls.
+This problem was solved by changing the topological sort to
+increase ignore_priority in order to select a smaller number
+of leaf nodes at a time.
"""
ebuilds = {
"app-alternatives/gzip-1": {
diff --git a/lib/portage/tests/resolver/test_merge_order.py
b/lib/portage/tests/resolver/test_merge_order.py
index 940eb3bbbe..e6d45c847b 100644
--- a/lib/portage/tests/resolver/test_merge_order.py
+++ b/lib/portage/tests/resolver/test_merge_order.py
@@ -1,4 +1,4 @@
-# Copyright 2011-2020 Gentoo Authors
+# Copyright 2011-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -382,10 +382,12 @@ class MergeOrderTestCase(TestCase):
ambiguous_merge_order=True,
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 2e298ea7ba36801a1cfba6e4cbfc16a7c05ee73d
Author: Sam James gentoo org>
AuthorDate: Tue Nov 28 06:22:44 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue Nov 28 22:07:46 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2e298ea7
DepPriority{Normal,Satisfied}Range: strengthen _ignore_runtime for runtime slot
operators
In the reported bug, net-misc/curl gets merged (binary), then dev-util/cmake
gets
bulit (from source) which fails because one of the built curl's dependencies
(net-libs/nghttp2) is missing:
```
[binary R] net-misc/curl-8.4.0::test_repo USE="http2%*" 0 KiB
[ebuild U ] dev-util/cmake-3.27.8::test_repo [3.26.5-r2::test_repo] 0 KiB
[ebuild N ] net-libs/nghttp2-1.57.0::test_repo 0 KiB
```
Zac had the idea [0] of strengthening _ignore_runtime to consider runtime slot
deps
as well, so we now get:
```
[ebuild U ] dev-util/cmake-3.27.8::test_repo [3.26.5-r2::test_repo] 0 KiB
[ebuild N ] net-libs/nghttp2-1.57.0::test_repo 0 KiB
[binary R] net-misc/curl-8.4.0::test_repo USE="http2%*" 0 KiB
```
For DepPrioritySatisfiedRange, we now allow ignoring the dep if:
* it's either a satisfied runtime slot dep, or it's not a runtime slot dep at
all, and
* the dep is satisfied or it's optional/not a build time dep.
(i.e. we now prevent ignoring the slot dep unless it's satisfied.)
For DepPriorityNormalRange, we now allow ignoring the dep if:
* it's not a runtime slot dep, and
* it's optional, or
* it's not a buildtime dep.
(i.e. we now prevent ignoring the slot dep.)
We then realise we can't ignore curl's dep on nghttp2 and come up with a better
order.
[0] https://github.com/gentoo/portage/pull/1193#issuecomment-1829178126
Bug: https://bugs.gentoo.org/918683
Thanks-to: Zac Medico gentoo.org>
Signed-off-by: Sam James gentoo.org>
lib/_emerge/DepPriorityNormalRange.py | 8 +++-
lib/_emerge/DepPrioritySatisfiedRange.py| 13 +++--
.../tests/resolver/test_runtime_cycle_merge_order.py| 9 -
3 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/lib/_emerge/DepPriorityNormalRange.py
b/lib/_emerge/DepPriorityNormalRange.py
index a85e1b9c14..d7e4381b47 100644
--- a/lib/_emerge/DepPriorityNormalRange.py
+++ b/lib/_emerge/DepPriorityNormalRange.py
@@ -37,7 +37,13 @@ class DepPriorityNormalRange:
def _ignore_runtime(cls, priority):
if priority.__class__ is not DepPriority:
return False
-return bool(priority.optional or not priority.buildtime)
+# If we ever allow "optional" runtime_slot_op, we'll need
+# to adjust this appropriately. But only build time dependencies
+# are optional right now, so it's not an issue as-is.
+return bool(
+not priority.runtime_slot_op
+and (priority.optional or not priority.buildtime)
+)
ignore_medium = _ignore_runtime
ignore_medium_soft = _ignore_runtime_post
diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py
b/lib/_emerge/DepPrioritySatisfiedRange.py
index 0633a5e1c2..0d42e7613d 100644
--- a/lib/_emerge/DepPrioritySatisfiedRange.py
+++ b/lib/_emerge/DepPrioritySatisfiedRange.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2013 Gentoo Foundation
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.DepPriority import DepPriority
@@ -89,7 +89,16 @@ class DepPrioritySatisfiedRange:
def _ignore_runtime(cls, priority):
if priority.__class__ is not DepPriority:
return False
-return bool(priority.satisfied or priority.optional or not
priority.buildtime)
+# We could split this up into 2 variants (ignoring satisfied
+# runtime_slot_op, and not) if we need more granularity for
ignore_priority
+# in future.
+return bool(
+(
+(not priority.runtime_slot_op)
+or (priority.satisfied and priority.runtime_slot_op)
+)
+and (priority.satisfied or priority.optional or not
priority.buildtime)
+)
ignore_medium = _ignore_runtime
ignore_medium_soft = _ignore_satisfied_buildtime_slot_op
diff --git a/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
b/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
index 26850ccad2..ed329aa097 100644
--- a/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
+++ b/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
@@ -1,4 +1,4 @@
-# Copyright 2016 Gentoo Foundation
+# Copyright 2016-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -7,8 +7,6 @@ from portage.tests.resolver.ResolverPlayground import (
ResolverPlaygroundTestCase,
)
-import pytest
-
class RuntimeCycleMergeOrderTestCase(TestCase):
def testRuntimeCycleMergeOrder(self):
@@ -77,7
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 49e01d041c74680a81860b819daff812d83df02f Author: Zac Medico gentoo org> AuthorDate: Tue Nov 28 03:42:17 2023 + Commit: Zac Medico gentoo org> CommitDate: Tue Nov 28 03:42:17 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=49e01d04 find_smallest_cycle: Optimize to traverse fewer nodes If gather_deps is traversing a cycle that is greater than or equal to the size of the current smallest_cycle, then abort early. Also abort early if we traverse a node encountered in a previous cycle for the same ignore_priority, since that means the two cycles are identical. On my laptop, this brings the emerge -pe @world time down to 3m28.884s, compared to 10m44.268s with portage-3.0.55. Bug: https://bugs.gentoo.org/918682 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/depgraph.py| 36 -- .../tests/resolver/test_rebuild_ghostscript.py | 2 +- .../resolver/test_runtime_cycle_merge_order.py | 7 +++-- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index e4305c18c9..29eadba3d1 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -9151,7 +9151,14 @@ class depgraph: asap_nodes.extend(libc_pkgs) -def gather_deps(ignore_priority, mergeable_nodes, selected_nodes, node): +def gather_deps( +ignore_priority, +mergeable_nodes, +selected_nodes, +node, +smallest_cycle=None, +traversed_nodes=None, +): """ Recursively gather a group of nodes that RDEPEND on eachother. This ensures that they are merged as a group @@ -9171,10 +9178,24 @@ class depgraph: # RDEPENDs installed first, but ignore uninstalls # (these occur when new portage blocks an older package version). return False +if traversed_nodes is not None: +if node in traversed_nodes: +# Identical to a previously traversed cycle. +return False +traversed_nodes.add(node) selected_nodes.add(node) +if smallest_cycle is not None and len(selected_nodes) >= len( +smallest_cycle +): +return False for child in mygraph.child_nodes(node, ignore_priority=ignore_priority): if not gather_deps( -ignore_priority, mergeable_nodes, selected_nodes, child +ignore_priority, +mergeable_nodes, +selected_nodes, +child, +smallest_cycle=smallest_cycle, +traversed_nodes=traversed_nodes, ): return False return True @@ -9332,12 +9353,21 @@ class depgraph: local_priority_range.MEDIUM_SOFT + 1, ) ): +# Traversed nodes for current priority +traversed_nodes = set() for node in nodes: if not mygraph.parent_nodes(node): continue +if node in traversed_nodes: +continue selected_nodes = set() if gather_deps( -priority, mergeable_nodes, selected_nodes, node +priority, +mergeable_nodes, +selected_nodes, +node, +smallest_cycle=smallest_cycle, +traversed_nodes=traversed_nodes, ): if smallest_cycle is None or len(selected_nodes) < len( smallest_cycle diff --git a/lib/portage/tests/resolver/test_rebuild_ghostscript.py b/lib/portage/tests/resolver/test_rebuild_ghostscript.py index 8d7cbb1f92..88dc2b5fc3 100644 --- a/lib/portage/tests/resolver/test_rebuild_ghostscript.py +++ b/lib/portage/tests/resolver/test_rebuild_ghostscript.py @@ -137,9 +137,9 @@ class RebuildGhostscriptTestCase(TestCase): mergelist=[ "sys-apps/dbus-1.15.6", "x11-libs/gtk+-3.24.38", +"app-text/ghostscript-gpl-10.01.2", "net-print/cups-2.4.6", "net-dns/avahi-0.8-r7", -"app-text/ghostscript-gpl-10.01.2", "app-text/libspectre-0.2.12", "x11-libs/goffice-0.10.55", ], diff --git a/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 9206d5a75ecd2d9ae0fe63e57d28aa8061b5927e
Author: Zac Medico gentoo org>
AuthorDate: Sat Nov 18 17:07:59 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Nov 19 04:15:22 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9206d5a7
find_smallest_cycle: Increase ignore_priority to find smaller cycles
Fix AlternativesGzipTestCase by increasing ignore_priority in
order to find smaller cycles. This causes some changes in merge
order for MergeOrderTestCase, but they appear to be acceptable
since they prevent temporarily unsatisfied RDEPEND by relying
on satisfied installed build-time dependencies.
Add a test case for bug 690436, since the change to merge order
in MergeOrderTestCase is related (see commit 680276cc4d4f).
Bug: https://bugs.gentoo.org/690436
Bug: https://bugs.gentoo.org/917259
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py| 7 +-
lib/portage/tests/resolver/meson.build | 1 +
.../tests/resolver/test_alternatives_gzip.py | 7 +-
lib/portage/tests/resolver/test_merge_order.py | 24 ++--
.../tests/resolver/test_rebuild_ghostscript.py | 160 +
5 files changed, 180 insertions(+), 19 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index e4305c18c9..0d3b37c698 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -9345,9 +9345,10 @@ class depgraph:
smallest_cycle = selected_nodes
ignore_priority = priority
-# Exit this loop with the lowest possible priority,
which
-# minimizes the use of installed packages to break
cycles.
-if smallest_cycle is not None:
+if smallest_cycle is not None and len(smallest_cycle)
== 1:
+# The cycle can't get any smaller than this,
+# so there is no need to search further since
+# we try to minimize ignore_priority.
break
return smallest_cycle, ignore_priority
diff --git a/lib/portage/tests/resolver/meson.build
b/lib/portage/tests/resolver/meson.build
index 7d2bd367d4..770027ac47 100644
--- a/lib/portage/tests/resolver/meson.build
+++ b/lib/portage/tests/resolver/meson.build
@@ -49,6 +49,7 @@ py.install_sources(
'test_profile_default_eapi.py',
'test_profile_package_set.py',
'test_rebuild.py',
+'test_rebuild_ghostscript.py',
'test_regular_slot_change_without_revbump.py',
'test_required_use.py',
'test_runtime_cycle_merge_order.py',
diff --git a/lib/portage/tests/resolver/test_alternatives_gzip.py
b/lib/portage/tests/resolver/test_alternatives_gzip.py
index 602ed1756f..7cd1da25f1 100644
--- a/lib/portage/tests/resolver/test_alternatives_gzip.py
+++ b/lib/portage/tests/resolver/test_alternatives_gzip.py
@@ -1,8 +1,6 @@
# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import pytest
-
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
@@ -10,7 +8,6 @@ from portage.tests.resolver.ResolverPlayground import (
)
-@pytest.mark.xfail()
class AlternativesGzipTestCase(TestCase):
def testAlternativesGzip(self):
"""
@@ -19,8 +16,8 @@ class AlternativesGzipTestCase(TestCase):
find_smallest_cycle selects a large cycle and the topological
sort produces poor results when leaf_nodes returns
app-alternatives/gzip as part of a large group of nodes.
-This problem might be solved by implementing a finer-grained
-ignore_priority for leaf_nodes calls.
+This problem was solved by increasing ignore_priority in order
+to find a smaller cycle.
"""
ebuilds = {
"app-alternatives/gzip-1": {
diff --git a/lib/portage/tests/resolver/test_merge_order.py
b/lib/portage/tests/resolver/test_merge_order.py
index 940eb3bbbe..671543ca29 100644
--- a/lib/portage/tests/resolver/test_merge_order.py
+++ b/lib/portage/tests/resolver/test_merge_order.py
@@ -382,10 +382,12 @@ class MergeOrderTestCase(TestCase):
ambiguous_merge_order=True,
# The following merge order assertion reflects optimal order
for
# a circular relationship which is DEPEND in one direction and
-# RDEPEND in the other.
-merge_order_assertions=(
-("app-misc/circ-buildtime-a-1",
"app-misc/circ-buildtime-c-1"),
-),
+# RDEPEND in the other. However, it is not respected because
+# it would result in a temporarily broken RDEPEND, so we
instead
+# rely on satisfied installed build-time dependencies.
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/, /
commit: c56ca69564452198039e17b3a07f24f5b87a6852
Author: Sam James gentoo org>
AuthorDate: Mon Nov 6 15:03:10 2023 +
Commit: Sam James gentoo org>
CommitDate: Mon Nov 6 15:57:56 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c56ca695
emerge: fix _show_ignored_binaries_respect_use with incomplete depgraph
I've gone for the simpler solution of just using an empty tuple where the
merge list is empty to preserve prior behaviour with what we do (or do not)
display wrt skipped binaries.
Bug: https://bugs.gentoo.org/916614
Signed-off-by: Sam James gentoo.org>
NEWS| 2 +
lib/_emerge/depgraph.py | 4 +-
lib/portage/tests/resolver/test_useflags.py | 89 +
3 files changed, 94 insertions(+), 1 deletion(-)
diff --git a/NEWS b/NEWS
index 4a33559a33..89d9335275 100644
--- a/NEWS
+++ b/NEWS
@@ -15,6 +15,8 @@ Bug fixes:
* Avoid crash with incomplete depgraph for binpkg-respect-use notice (bug
#916614).
+* Avoid crash with blockers in depgraph for binpkg-respect-use notice (bug
#916336).
+
portage-3.0.54 (2023-10-25)
--
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index d1fed0d652..e4305c18c9 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1234,11 +1234,13 @@ class depgraph:
def _show_ignored_binaries_respect_use(self, respect_use):
seen = {}
messages = []
+
merging = {
(pkg.root, pkg.cpv)
-for pkg in self._dynamic_config._displayed_list
+for pkg in self._dynamic_config._displayed_list or ()
if isinstance(pkg, Package)
}
+
for pkg, flags in respect_use.items():
# Don't include recursive deps which aren't in the merge list
anyway.
if (pkg.root, pkg.cpv) not in merging:
diff --git a/lib/portage/tests/resolver/test_useflags.py
b/lib/portage/tests/resolver/test_useflags.py
index ee9f4f2763..86684f7f28 100644
--- a/lib/portage/tests/resolver/test_useflags.py
+++ b/lib/portage/tests/resolver/test_useflags.py
@@ -229,3 +229,92 @@ class UseFlagsTestCase(TestCase):
)
finally:
playground.cleanup()
+
+def testNoMergeBinpkgRespectUse(self):
+"""
+Testcase for bug #916614 where an incomplete depgraph may be fed into
+_show_ignored_binaries_respect_use.
+
+We use a mix of +/-abi_x86_32 to trigger the binpkg-respect-use notice
+and depend on a non-existent package in one of the available ebuilds we
+queue to reinstall to trigger an aborted calculation.
+"""
+ebuilds = {
+"dev-libs/A-2": {
+"EAPI": "7",
+"IUSE": "abi_x86_32",
+},
+"dev-libs/B-1": {
+"IUSE": "abi_x86_32",
+"RDEPEND": "=dev-libs/A-1",
+},
+}
+
+installed = {
+"dev-libs/B-1": {
+"IUSE": "abi_x86_32",
+"USE": "abi_x86_32",
+},
+"dev-libs/A-1": {
+"IUSE": "abi_x86_32",
+"USE": "abi_x86_32",
+},
+}
+
+binpkgs = {
+"dev-libs/A-2": {
+"IUSE": "abi_x86_32",
+"USE": "abi_x86_32",
+},
+"dev-libs/B-1": {
+"IUSE": "abi_x86_32",
+"USE": "",
+"BUILD_ID": "2",
+"BUILD_TIME": "2",
+},
+}
+
+user_config = {
+"make.conf": (
+'FEATURES="binpkg-multi-instance"',
+'USE="abi_x86_32 abi_x86_32"',
+),
+}
+
+world = ("dev-libs/A",)
+
+test_cases = (
+ResolverPlaygroundTestCase(
+["@installed"],
+options={
+"--verbose": "y",
+"--emptytree": True,
+"--usepkg": True,
+},
+success=False,
+mergelist=["[binary]dev-libs/A-2", "dev-libs/B-1"],
+slot_collision_solutions=[],
+),
+)
+
+for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+with self.subTest(binpkg_format=binpkg_format):
+print(colorize("HILITE", binpkg_format), end=" ... ")
+sys.stdout.flush()
+user_config["make.conf"] +=
(f'BINPKG_FORMAT="{binpkg_format}"',)
+playground = ResolverPlayground(
+ebuilds=ebuilds,
+binpkgs=binpkgs,
+installed=installed,
+user_config=user_config,
+world=world,
+)
+
+try:
+for test_case in test_cases:
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/, /
commit: 262b8c9213c6698be4a7e2ce88536568899e6c2a
Author: Sam James gentoo org>
AuthorDate: Mon Oct 30 05:30:04 2023 +
Commit: Sam James gentoo org>
CommitDate: Mon Nov 6 15:57:56 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=262b8c92
emerge: fix binpkg-respect-use notice with blockers
Items in _dynamic_config._displayed_list might be blockers, so filter those out.
Bug: https://bugs.gentoo.org/916336
Fixes: bb82666b48e18f448661a1a8bf6a39b773cc4b1c
Signed-off-by: Sam James gentoo.org>
NEWS| 2 +
lib/_emerge/depgraph.py | 6 +-
lib/portage/tests/resolver/test_useflags.py | 87 +
3 files changed, 94 insertions(+), 1 deletion(-)
diff --git a/NEWS b/NEWS
index be551de33d..4a33559a33 100644
--- a/NEWS
+++ b/NEWS
@@ -13,6 +13,8 @@ Bug fixes:
* Convert portageq helper to a function to avoid breaking external callers
(bug #916287, bug #916296).
+* Avoid crash with incomplete depgraph for binpkg-respect-use notice (bug
#916614).
+
portage-3.0.54 (2023-10-25)
--
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 85845dc1e2..0717e0429f 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1234,7 +1234,11 @@ class depgraph:
def _show_ignored_binaries_respect_use(self, respect_use):
seen = {}
messages = []
-merging = {pkg.cpv for pkg in self._dynamic_config._displayed_list}
+merging = {
+pkg.cpv
+for pkg in self._dynamic_config._displayed_list
+if isinstance(pkg, Package)
+}
for pkg, flags in respect_use.items():
# Don't include recursive deps which aren't in the merge list
anyway.
if pkg.cpv not in merging:
diff --git a/lib/portage/tests/resolver/test_useflags.py
b/lib/portage/tests/resolver/test_useflags.py
index 0af1cb5585..ee9f4f2763 100644
--- a/lib/portage/tests/resolver/test_useflags.py
+++ b/lib/portage/tests/resolver/test_useflags.py
@@ -142,3 +142,90 @@ class UseFlagsTestCase(TestCase):
)
finally:
playground.cleanup()
+
+def testBlockerBinpkgRespectUse(self):
+"""
+Test for bug #916336 where we tried to check properties of a blocker
+object which isn't a Package to be merged.
+"""
+
+ebuilds = {
+"dev-libs/A-1": {
+"EAPI": "7",
+"IUSE": "abi_x86_32",
+"RDEPEND": "dev-libs/B",
+},
+"dev-libs/B-1": {
+"EAPI": "7",
+"IUSE": "abi_x86_32",
+},
+"dev-libs/A-2": {
+"EAPI": "7",
+"IUSE": "abi_x86_32",
+"RDEPEND": "!
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/process/, lib/portage/util/_async/
commit: 9a56879ec26bfb39dc3551932d664fd6bec80500
Author: Zac Medico gentoo org>
AuthorDate: Sun Oct 22 16:46:22 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Oct 22 17:42:26 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9a56879e
SpawnProcess/ForkProcess: Pass stdin via fd_pipes if not background
Bug: https://bugs.gentoo.org/916116
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/SpawnProcess.py | 2 +-
lib/portage/tests/process/test_AsyncFunction.py | 33 -
lib/portage/util/_async/ForkProcess.py | 2 +-
3 files changed, 24 insertions(+), 13 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index b4eabd07ad..72fa72c613 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -62,7 +62,7 @@ class SpawnProcess(SubProcess):
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
-if fd_pipes or self.logfile:
+if fd_pipes or self.logfile or not self.background:
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
diff --git a/lib/portage/tests/process/test_AsyncFunction.py
b/lib/portage/tests/process/test_AsyncFunction.py
index e30ff770b3..d02a3395c1 100644
--- a/lib/portage/tests/process/test_AsyncFunction.py
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -16,25 +16,36 @@ from portage.util.futures.unix_events import
_set_nonblocking
class AsyncFunctionTestCase(TestCase):
@staticmethod
-def _read_from_stdin(pr, pw):
+def _read_from_stdin(pw):
if pw is not None:
os.close(pw)
-os.dup2(pr.fileno(), sys.stdin.fileno())
return "".join(sys.stdin)
async def _testAsyncFunctionStdin(self, loop):
test_string = "1\n2\n3\n"
pr, pw = multiprocessing.Pipe(duplex=False)
-reader = AsyncFunction(
-scheduler=loop,
-target=self._read_from_stdin,
-args=(
-pr,
-pw.fileno() if multiprocessing.get_start_method() == "fork"
else None,
-),
-)
-reader.start()
+stdin_backup = os.dup(portage._get_stdin().fileno())
+os.dup2(pr.fileno(), portage._get_stdin().fileno())
pr.close()
+try:
+reader = AsyncFunction(
+# Should automatically inherit stdin as fd_pipes[0]
+# when background is False, for things like
+# emerge --sync --ask (bug 916116).
+background=False,
+scheduler=loop,
+target=self._read_from_stdin,
+args=(
+pw.fileno()
+if multiprocessing.get_start_method() == "fork"
+else None,
+),
+)
+reader.start()
+finally:
+os.dup2(stdin_backup, portage._get_stdin().fileno())
+os.close(stdin_backup)
+
_set_nonblocking(pw.fileno())
with open(pw.fileno(), mode="wb", buffering=0, closefd=False) as
pipe_write:
await _writer(pipe_write, test_string.encode("utf_8"))
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index 6d216a5c43..780545be0e 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -52,7 +52,7 @@ class ForkProcess(SpawnProcess):
'fd_pipes only supported with HAVE_SEND_HANDLE or
multiprocessing start method "fork"'
)
-if self.fd_pipes or self.logfile:
+if self.fd_pipes or self.logfile or not self.background:
# Log via multiprocessing.Pipe if necessary.
connection, self._child_connection = multiprocessing.Pipe(
duplex=self._HAVE_SEND_HANDLE
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/ebuild/
commit: 919567592eb2292def12877420f280e082cc881a
Author: Zac Medico gentoo org>
AuthorDate: Tue Oct 3 19:07:37 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Oct 3 19:43:08 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=91956759
SpawnProcess: Make fd_pipes optional
Since fd_pipes is incompatible with the multiprocessing spawn start
method, allow empty fd_pipes so that ForkProcess can inherit
SpawnProcess and still be compatible with multiprocessing spawn.
Bug: https://bugs.gentoo.org/915128
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/SpawnProcess.py | 111 +++-
lib/portage/tests/ebuild/test_ipc_daemon.py | 9 ++-
2 files changed, 68 insertions(+), 52 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 8f21f05c04..0e27f29c43 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2008-2021 Gentoo Authors
+# Copyright 2008-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -62,55 +62,60 @@ class SpawnProcess(SubProcess):
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
-master_fd, slave_fd = self._pipe(fd_pipes)
-
-can_log = self._can_log(slave_fd)
-if can_log:
-log_file_path = self.logfile
-else:
-log_file_path = None
-
-null_input = None
-if not self.background or 0 in fd_pipes:
-# Subclasses such as AbstractEbuildProcess may have already passed
-# in a null file descriptor in fd_pipes, so use that when given.
-pass
+if fd_pipes or self.logfile:
+master_fd, slave_fd = self._pipe(fd_pipes)
+
+can_log = self._can_log(slave_fd)
+if can_log:
+log_file_path = self.logfile
+else:
+log_file_path = None
+
+null_input = None
+if not self.background or 0 in fd_pipes:
+# Subclasses such as AbstractEbuildProcess may have already
passed
+# in a null file descriptor in fd_pipes, so use that when
given.
+pass
+else:
+# TODO: Use job control functions like tcsetpgrp() to control
+# access to stdin. Until then, use /dev/null so that any
+# attempts to read from stdin will immediately return EOF
+# instead of blocking indefinitely.
+null_input = os.open("/dev/null", os.O_RDWR)
+fd_pipes[0] = null_input
+
+fd_pipes.setdefault(0, portage._get_stdin().fileno())
+fd_pipes.setdefault(1, sys.__stdout__.fileno())
+fd_pipes.setdefault(2, sys.__stderr__.fileno())
+
+# flush any pending output
+stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
+for fd in fd_pipes.values():
+if fd in stdout_filenos:
+sys.__stdout__.flush()
+sys.__stderr__.flush()
+break
+
+fd_pipes_orig = fd_pipes.copy()
+
+if log_file_path is not None or self.background:
+fd_pipes[1] = slave_fd
+fd_pipes[2] = slave_fd
+
+else:
+# Create a dummy pipe that PipeLogger uses to efficiently
+# monitor for process exit by listening for the EOF event.
+# Re-use of the allocated fd number for the key in fd_pipes
+# guarantees that the keys will not collide for similarly
+# allocated pipes which are used by callers such as
+# FileDigester and MergeProcess. See the _setup_pipes
+# docstring for more benefits of this allocation approach.
+self._dummy_pipe_fd = slave_fd
+fd_pipes[slave_fd] = slave_fd
else:
-# TODO: Use job control functions like tcsetpgrp() to control
-# access to stdin. Until then, use /dev/null so that any
-# attempts to read from stdin will immediately return EOF
-# instead of blocking indefinitely.
-null_input = os.open("/dev/null", os.O_RDWR)
-fd_pipes[0] = null_input
-
-fd_pipes.setdefault(0, portage._get_stdin().fileno())
-fd_pipes.setdefault(1, sys.__stdout__.fileno())
-fd_pipes.setdefault(2, sys.__stderr__.fileno())
-
-# flush any pending output
-stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
-for fd in fd_pipes.values():
-if fd in stdout_filenos:
-sys.__stdout__.flush()
-sys.__stderr__.flush()
-break
-
-fd_pipes_orig = fd_pipes.copy()
-
-if log_file_path is not None or self.background:
-
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 224207c7d1988a354d004507bb7ecfb90b4ef097 Author: YiFei Zhu gmail com> AuthorDate: Tue Jun 13 00:47:52 2023 + Commit: Sam James gentoo org> CommitDate: Fri Jun 16 03:34:46 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=224207c7 depgraph: Do not allow slotted deps to be satisfied by wrong slots This may be part of what caused the "perl rebuild bug". The "priority" of perl, when seen by perl modules, may have "satisfied" set to an installed perl of a wrong slot. Compounding this factor with bug #756199 where find_smallest_cycle would select a single node, in this case because the dependency of perl is satisfied and the priority then gets ignored, the "cycle" becomes a perl module alone and gets rebuilt early. I also updated the test from the previous patch to account for this change. No other tests seems affected. For a larger scale test, I reproduced this initially with a stage3 chroot from a Jan 1 2022 stage3 snapshot, and testing in an equivalent dockerfile would work too: FROM gentoo/stage3:amd64-openrc-20220101 RUN emerge-webrsync COPY . /portage Before this patch (USE flags omitted): # cd /portage && bin/emerge -puDN @world 2>&1 | grep -i perl [ebuild U ] app-admin/perl-cleaner-2.30-r1 [2.30] [ebuild rR] virtual/perl-File-Temp-0.231.100 [ebuild rR] dev-perl/Locale-gettext-1.70.0-r1 [ebuild rR] dev-perl/MIME-Charset-1.12.2-r1 [ebuild rR] dev-perl/Module-Build-0.423.100 [ebuild rR] dev-perl/Text-CharWidth-0.40.0-r2 [ebuild U ] dev-lang/perl-5.36.0-r2 [5.34.0-r3] [ebuild N ] virtual/perl-CPAN-2.330.0 [ebuild U ] virtual/perl-ExtUtils-MakeMaker-7.640.0 [7.620.0] [ebuild U ] virtual/perl-File-Spec-3.840.0 [3.800.0] [...] After this patch: # cd /portage && bin/emerge -puDN @world 2>&1 | grep -i perl [ebuild U ] app-admin/perl-cleaner-2.30-r1 [2.30] [ebuild U ] dev-lang/perl-5.36.0-r2:0/5.36 [5.34.0-r3:0/5.34] [ebuild N ] virtual/perl-CPAN-2.330.0 [ebuild U ] virtual/perl-ExtUtils-MakeMaker-7.640.0 [7.620.0] [ebuild U ] virtual/perl-Data-Dumper-2.184.0 [2.179.0] [ebuild U ] virtual/perl-File-Spec-3.840.0 [3.800.0] [ebuild U ] virtual/perl-Test-Harness-3.440.0-r1 [3.430.0] [ebuild rR] dev-perl/Pod-Parser-1.630.0-r1 [ebuild rR] dev-perl/Text-CharWidth-0.40.0-r2 [ebuild rR] dev-perl/Text-WrapI18N-0.60.0-r2 [...] Bug: https://bugs.gentoo.org/463976 Bug: https://bugs.gentoo.org/592880 Bug: https://bugs.gentoo.org/596664 Bug: https://bugs.gentoo.org/631490 Bug: https://bugs.gentoo.org/764365 Bug: https://bugs.gentoo.org/793992 Signed-off-by: YiFei Zhu gmail.com> Closes: https://github.com/gentoo/portage/pull/1055 Signed-off-by: Sam James gentoo.org> lib/_emerge/depgraph.py | 18 ++ lib/portage/tests/resolver/test_perl_rebuild_bug.py | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 60e57b226..28acfed9d 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -4042,6 +4042,16 @@ class depgraph: inst_pkg, modified_use=self._pkg_use_enabled(inst_pkg) ) ] +# Do not allow slotted deps to be satisfied by wrong slots. +# Otherwise, slot-operator-dependent packages may rebuild +# before the slotted package they are dependent on. +if child and atom.slot_operator == "=": +inst_pkgs = [ +inst_pkg +for inst_pkg in inst_pkgs +if inst_pkg.slot == child.slot +and inst_pkg.sub_slot == child.sub_slot +] if inst_pkgs: for inst_pkg in inst_pkgs: if self._pkg_visibility_check(inst_pkg): @@ -4161,6 +4171,14 @@ class depgraph: inst_pkg, modified_use=self._pkg_use_enabled(inst_pkg) ) ] +# Do not allow slotted deps to be satisfied by wrong slots. +if child and atom.slot_operator == "=": +inst_pkgs = [ +inst_pkg +for inst_pkg in inst_pkgs +if inst_pkg.slot == child.slot +and inst_pkg.sub_slot == child.sub_slot +] if inst_pkgs: for inst_pkg in inst_pkgs: if self._pkg_visibility_check(inst_pkg): diff --git a/lib/portage/tests/resolver/test_perl_rebuild_bug.py b/lib/portage/tests/resolver/test_perl_rebuild_bug.py index 928fd47d7..7e376f396 100644 --- a/lib/portage/tests/resolver/test_perl_rebuild_bug.py +++
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 44afa8445dc46464200fe46c1e09e0c7475067bf
Author: YiFei Zhu gmail com>
AuthorDate: Mon Jun 12 02:23:09 2023 +
Commit: Sam James gentoo org>
CommitDate: Fri Jun 16 03:34:46 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=44afa844
depgraph: Don't ignore downgrades as missed_updates
Missed updates can also come in the form of package downgrades,
when, for example, there are keyword changes. They can cause
rebuilds, and these rebuilds may be not possible due to reasons such
as keywords or masks. In this case, prior to this patch, portage
would cancel the downgrade, but the rebuilds would be requested
endlessly, because bug 439688's backtrack code does not trigger.
To reproduce, on an ACCEPT_KEYWORDS=~amd64 machine, emerge
=dev-libs/openssl=3.0.9, dev-util/rustup, and something else that
depends on openssl. Then un-accept ~amd64 for openssl and rustup.
Prior to this patch, a @world upgrade would cause:
These are the packages that would be merged, in order:
Calculating dependencies... done!
[ebuild rR] dev-libs/libevent-2.1.12-r1:0/2.1-7::gentoo
[ebuild rR] net-misc/rsync-3.2.7-r2::gentoo
[...]
Total: 71 packages (71 reinstalls), Size of downloads: 0 KiB
There are no packages marked "R", only "rR". There are no section
labeled "The following packages are causing rebuilds:" either.
After this patch, we have:
These are the packages that would be merged, in order:
Calculating dependencies... done!
Total: 0 packages, Size of downloads: 0 KiB
WARNING: One or more updates/rebuilds have been skipped due to a dependency
conflict:
dev-libs/openssl:0
(dev-libs/openssl-1.1.1u:0/1.1::gentoo, ebuild scheduled for merge)
dev-libs/openssl:0/3= required by (dev-util/rustup-1.25.2:0/0::gentoo,
installed)
I also updated the test from the previous patch to account for
this change. No other tests seems affected.
Bug: https://bugs.gentoo.org/439688
Bug: https://bugs.gentoo.org/622270
Signed-off-by: YiFei Zhu gmail.com>
Closes: https://github.com/gentoo/portage/pull/1053
Signed-off-by: Sam James gentoo.org>
lib/_emerge/depgraph.py| 4 +---
lib/portage/tests/resolver/test_slot_conflict_blocked_prune.py | 2 +-
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 77133e99c..60e57b226 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1287,9 +1287,7 @@ class depgraph:
pkg.root, pkg.slot_atom
):
any_selected = True
-if chosen_pkg > pkg or (
-not chosen_pkg.installed and chosen_pkg.version ==
pkg.version
-):
+if not chosen_pkg.installed and chosen_pkg.version ==
pkg.version:
missed_update = False
break
if any_selected and missed_update:
diff --git a/lib/portage/tests/resolver/test_slot_conflict_blocked_prune.py
b/lib/portage/tests/resolver/test_slot_conflict_blocked_prune.py
index 14e98cd00..b23126d5f 100644
--- a/lib/portage/tests/resolver/test_slot_conflict_blocked_prune.py
+++ b/lib/portage/tests/resolver/test_slot_conflict_blocked_prune.py
@@ -63,7 +63,7 @@ class SlotConflictBlockedPruneTestCase(TestCase):
["@world"],
options={"--deep": True, "--update": True, "--verbose": True},
success=True,
-mergelist=["x11-base/xwayland-23.1.1"],
+mergelist=[],
),
)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/, /, man/
commit: 75a4f2c2e07c128fef4d9faf3a8fb9d67565239e
Author: Sam James gentoo org>
AuthorDate: Thu Feb 16 06:26:07 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue Feb 21 07:16:27 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=75a4f2c2
emerge: add --update-if-installed
This adds a new emerge option '--update-if-installed'.
The use case for such an option is as follows:
- User finds out libfoo-1.2 is buggy.
- They want to upgrade all their systems if libfoo is installed.
- They don't want to install libfoo if it's not already installed.
Unfortunately, --update fails this last point, hence
the need for a new option.
Closes: https://github.com/gentoo/portage/pull/988
Signed-off-by: Sam James gentoo.org>
NEWS | 4 ++
lib/_emerge/create_depgraph_params.py | 6 ++
lib/_emerge/depgraph.py | 17 -
lib/_emerge/main.py | 3 +-
lib/portage/tests/resolver/test_update.py | 106 ++
man/emerge.1 | 6 ++
6 files changed, 140 insertions(+), 2 deletions(-)
diff --git a/NEWS b/NEWS
index 29e9de038..9389d2c09 100644
--- a/NEWS
+++ b/NEWS
@@ -10,6 +10,10 @@ Features:
* emerge: add --onlydeps-with-ideps= option (bug #890777)
+* emerge: add --update-if-installed option. This is useful for one-shot
+ emerge commands to be run across several machines to upgrade packages
+ only if they're installed.
+
* install-qa-check.d: 60pkgconfig: add opt-in QA_PKGCONFIG_VERSION check
* emerge: Log completion of package installs.
diff --git a/lib/_emerge/create_depgraph_params.py
b/lib/_emerge/create_depgraph_params.py
index 531230402..1bbca5de9 100644
--- a/lib/_emerge/create_depgraph_params.py
+++ b/lib/_emerge/create_depgraph_params.py
@@ -129,6 +129,12 @@ def create_depgraph_params(myopts, myaction):
if changed_slot:
myparams["changed_slot"] = True
+# --update-if-installed implies --update
+update_if_installed = myopts.get("--update-if-installed")
+if update_if_installed is not None:
+myparams["update_if_installed"] = update_if_installed
+myopts["--update"] = True
+
if (
"--update" in myopts
or "--newrepo" in myopts
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 1631ed126..412dc7b6f 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -5019,6 +5019,21 @@ class depgraph:
pkg, existing_node = self._select_package(
myroot, atom, onlydeps=onlydeps
)
+
+# Is the package installed (at any version)?
+if pkg and "update_if_installed" in
self._dynamic_config.myparams:
+package_is_installed = any(
+self._iter_match_pkgs(
+self._frozen_config.roots[myroot],
"installed", atom
+)
+)
+
+# This package isn't eligible for selection in the
+# merge list as the user passed --update-if-installed
+# and it isn't installed.
+if not package_is_installed:
+continue
+
if not pkg:
pprovided_match = False
for virt_choice in virtuals.get(atom.cp, []):
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 38233e05c..850487d36 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import argparse
@@ -52,6 +52,7 @@ options = [
"--tree",
"--unordered-display",
"--update",
+"--update-if-installed",
]
shortmapping = {
diff --git a/lib/portage/tests/resolver/test_update.py
b/lib/portage/tests/resolver/test_update.py
new file mode 100644
index 0..e67013f9f
--- /dev/null
+++ b/lib/portage/tests/resolver/test_update.py
@@ -0,0 +1,106 @@
+# Copyright 2022-2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+ResolverPlayground,
+ResolverPlaygroundTestCase,
+)
+
+
+class UpdateIfInstalledTestCase(TestCase):
+def testUpdateIfInstalledEmerge(self):
+installed = {
+"dev-lang/ghc-4": {},
+"dev-libs/larryware-3": {},
+"dev-libs/larryware-ng-3": {},
+"virtual/libc-1": {},
+}
+
+ebuilds = installed.copy()
+ebuilds.update(
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, /, lib/portage/elog/
commit: a28764138198e641cd0fdcc166d596f2cd076225
Author: James Le Cuirot gentoo org>
AuthorDate: Tue Dec 13 21:28:22 2022 +
Commit: Sam James gentoo org>
CommitDate: Thu Dec 15 00:50:37 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a2876413
logging: Write logs under BROOT, not EPREFIX
Logs should be written to the build system, not the host system. When
building for a prefixed system under ROOT, logs were being written under
/${EPREFIX} even though that directory did not exist at all.
Signed-off-by: James Le Cuirot gentoo.org>
Closes: https://github.com/gentoo/portage/pull/959
Signed-off-by: Sam James gentoo.org>
NEWS | 2 ++
lib/_emerge/actions.py | 5 -
lib/portage/elog/mod_save.py | 2 +-
lib/portage/elog/mod_save_summary.py | 2 +-
4 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/NEWS b/NEWS
index 4684487b3..e80c82227 100644
--- a/NEWS
+++ b/NEWS
@@ -15,6 +15,8 @@ Bug fixes:
* make.conf(5): Note the various file suffixes / file extensions used
for binary packages.
+* logging: Write logs under BROOT, not EPREFIX.
+
portage-3.0.41 (2022-11-04)
--
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 30b687140..047948f56 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -3799,7 +3799,10 @@ def run_action(emerge_config):
emerge_log_dir = emerge_config.target_config.settings.get("EMERGE_LOG_DIR")
default_log_dir = os.path.join(
-os.sep, portage.const.EPREFIX.lstrip(os.sep), "var", "log"
+os.sep,
+emerge_config.target_config.settings["BROOT"].lstrip(os.sep),
+"var",
+"log",
)
for x in ("--pretend", "--fetchonly", "--fetch-all-uri"):
if x in emerge_config.opts:
diff --git a/lib/portage/elog/mod_save.py b/lib/portage/elog/mod_save.py
index a2b2cf7bd..e526b331f 100644
--- a/lib/portage/elog/mod_save.py
+++ b/lib/portage/elog/mod_save.py
@@ -20,7 +20,7 @@ def process(mysettings, key, logentries, fulltext):
logdir = normalize_path(mysettings["PORTAGE_LOGDIR"])
else:
logdir = os.path.join(
-os.sep, mysettings["EPREFIX"].lstrip(os.sep), "var", "log",
"portage"
+os.sep, mysettings["BROOT"].lstrip(os.sep), "var", "log", "portage"
)
if not os.path.isdir(logdir):
diff --git a/lib/portage/elog/mod_save_summary.py
b/lib/portage/elog/mod_save_summary.py
index 5aa31a8e1..f2913da5b 100644
--- a/lib/portage/elog/mod_save_summary.py
+++ b/lib/portage/elog/mod_save_summary.py
@@ -20,7 +20,7 @@ def process(mysettings, key, logentries, fulltext):
logdir = normalize_path(mysettings["PORTAGE_LOGDIR"])
else:
logdir = os.path.join(
-os.sep, mysettings["EPREFIX"].lstrip(os.sep), "var", "log",
"portage"
+os.sep, mysettings["BROOT"].lstrip(os.sep), "var", "log", "portage"
)
if not os.path.isdir(logdir):
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, /, lib/portage/util/, lib/portage/, bin/, ...
commit: d0fa3d28c5de6a0a34ac87de5e1a463adbe58405
Author: Sam James gentoo org>
AuthorDate: Thu Sep 29 06:37:19 2022 +
Commit: Sam James gentoo org>
CommitDate: Tue Oct 11 19:18:35 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d0fa3d28
binpkg: compress/decompress xz in parallel; compress zstd in parallel
- As before, needs >= xz-5.3.3_alpha for parallel decompression.
- zstd does not support parallel decompression.
Closes: https://github.com/gentoo/portage/pull/918
Signed-off-by: Sam James gentoo.org>
NEWS | 7 +++
bin/quickpkg | 8 +++-
lib/_emerge/BinpkgExtractorAsync.py| 4
lib/portage/gpkg.py| 8 +++-
lib/portage/package/ebuild/doebuild.py | 14 +++---
lib/portage/util/compression_probe.py | 6 +++---
lib/portage/util/cpuinfo.py| 22 +-
7 files changed, 60 insertions(+), 9 deletions(-)
diff --git a/NEWS b/NEWS
index 28e8f97ca..a3591e94d 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,10 @@
+portage-3.0.39 (UNRELEASED)
+--
+
+Features:
+* gpkg: Call xz and zstd with -T N where N is the number of jobs from
${MAKEOPTS}
+ for parallel compression (xz & zstd) and decompression (xz with newer
versions).
+
portage-3.0.38.1 (2022-10-04)
--
diff --git a/bin/quickpkg b/bin/quickpkg
index 67eee697a..9ff8d7b5a 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -36,6 +36,7 @@ from portage.exception import (
PermissionDenied,
)
from portage.util import ensure_dirs, shlex_split, varexpand, _xattr
+from portage.util.cpuinfo import makeopts_to_job_count
xattr = _xattr.xattr
from portage._sets import load_default_config, SETPREFIX
@@ -168,7 +169,12 @@ def quickpkg_atom(options, infos, arg, eout):
% (binpkg_compression, missing_package)
)
return 1
-cmd = shlex_split(varexpand(compression["compress"],
mydict=settings))
+
+cmd = compression["compress"].replace(
+"{JOBS}",
+str(makeopts_to_job_count(settings.get("MAKEOPTS", "1"))),
+)
+cmd = shlex_split(varexpand(cmd, mydict=settings))
# Filter empty elements that make Popen fail
cmd = [x for x in cmd if x != ""]
with open(binpkg_tmpfile, "wb") as fobj:
diff --git a/lib/_emerge/BinpkgExtractorAsync.py
b/lib/_emerge/BinpkgExtractorAsync.py
index 919837fc1..65b383998 100644
--- a/lib/_emerge/BinpkgExtractorAsync.py
+++ b/lib/_emerge/BinpkgExtractorAsync.py
@@ -10,6 +10,7 @@ from portage.util.compression_probe import (
compression_probe,
_compressors,
)
+from portage.util.cpuinfo import makeopts_to_job_count
from portage.process import find_binary
from portage.util import (
shlex_split,
@@ -53,6 +54,9 @@ class BinpkgExtractorAsync(SpawnProcess):
decomp = _compressors.get(compression_probe(self.pkg_path))
if decomp is not None:
decomp_cmd = decomp.get("decompress")
+decomp_cmd = decomp_cmd.replace(
+"{JOBS}", str(makeopts_to_job_count(self.env.get("MAKEOPTS",
"1")))
+)
elif tarfile.is_tarfile(
portage._unicode_encode(
self.pkg_path, encoding=portage._encodings["fs"],
errors="strict"
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 7e1b98370..1f8fa1fd3 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -38,6 +38,7 @@ from portage.util._urlopen import urlopen
from portage.util import writemsg
from portage.util import shlex_split, varexpand
from portage.util.compression_probe import _compressors
+from portage.util.cpuinfo import makeopts_to_job_count
from portage.process import find_binary
from portage.const import MANIFEST2_HASH_DEFAULTS, HASHING_BLOCKSIZE
@@ -1787,7 +1788,12 @@ class gpkg:
if mode not in compressor:
raise InvalidCompressionMethod("{}: {}".format(compression, mode))
-cmd = shlex_split(varexpand(compressor[mode], mydict=self.settings))
+cmd = compressor[mode]
+cmd = cmd.replace(
+"{JOBS}", str(makeopts_to_job_count(self.settings.get("MAKEOPTS",
"1")))
+)
+cmd = shlex_split(varexpand(cmd, mydict=self.settings))
+
# Filter empty elements that make Popen fail
cmd = [x for x in cmd if x != ""]
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index d0d134b39..4f7049c21 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -111,7 +111,7 @@ from portage.util import (
writemsg_stdout,
write_atomic,
)
-from portage.util.cpuinfo import get_cpu_count
+from portage.util.cpuinfo import get_cpu_count, makeopts_to_job_count
from portage.util.lafilefixer import
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/emerge/
commit: 7fe37c7bbf6c42fb0b5ea5d8a431abf677df5822
Author: David Palao gmail com>
AuthorDate: Fri Sep 16 13:48:39 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Sun Sep 25 19:10:01 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7fe37c7b
test(actions): requiring that run_action calls binarytree.populate correctly
...which means that since this is the first time, populate uses
getbinpkg_refresh=True explicitly.
This step is a preparation for a follow-up little change in the API of
binarytree.populate
Bug: https://bugs.gentoo.org/864259
Signed-off-by: David Palao gmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
lib/_emerge/actions.py | 4 ++-
lib/portage/tests/emerge/test_actions.py | 45
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 26120ad6d..fbfc561ec 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -3510,7 +3510,9 @@ def run_action(emerge_config):
try:
mytrees["bintree"].populate(
-getbinpkgs="--getbinpkg" in emerge_config.opts, **kwargs
+getbinpkgs="--getbinpkg" in emerge_config.opts,
+getbinpkg_refresh=True,
+**kwargs
)
except ParseError as e:
writemsg(
diff --git a/lib/portage/tests/emerge/test_actions.py
b/lib/portage/tests/emerge/test_actions.py
new file mode 100644
index 0..014d39dd9
--- /dev/null
+++ b/lib/portage/tests/emerge/test_actions.py
@@ -0,0 +1,45 @@
+# Copyright 2022 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from unittest.mock import MagicMock, patch
+
+from _emerge.actions import run_action
+from portage.tests import TestCase
+
+
+class RunActionTestCase(TestCase):
+"""This class' purpose is to encompass UTs for ``actions.run_action``.
+Since that function is extremely long (at least on Sep. 2022;
+hopefully the situation gets better with the time), the tests in this
+``TestCase`` contain plenty of mocks/patches.
+Hopefully, with time and effort, the ``run_action`` function (and others
+in the module) are refactored to make testing easier and more robust.
+
+A side effect of the mocking approach is a strong dependency on the
+details of the implementation. That can be improved if functions
+are smaller and do a well defined small set of tasks. Another call to
+refactoring...
+If the implementation changes, the mocks can be adjusted to play its
+role.
+"""
+@patch("_emerge.actions.profile_check")
+@patch("_emerge.actions.adjust_configs")
+@patch("_emerge.actions.apply_priorities")
+def test_binary_trees_populate_called(
+self,
+papply,
+padjust,
+profile_ckeck):
+config = MagicMock()
+config.action = None
+config.opts = {"--quiet": True, "--usepkg": True}
+bt = MagicMock()
+tree = {"bintree": bt}
+trees = {"first": tree}
+config.trees = trees
+
+run_action(config)
+
+bt.populate.assert_called_once_with(
+getbinpkgs=False, getbinpkg_refresh=True
+)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/, bin/, lib/portage/tests/ebuild/
commit: 704bcd1581e49432f363f0eda648d58411775d7f
Author: Zac Medico gentoo org>
AuthorDate: Sun Nov 7 04:19:58 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Nov 10 23:10:53 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=704bcd15
EbuildIpcDaemon: fix lock permission race
Move ipc files to a .ipc subdirectory, with a setgid bit to
prevent a lockfile group permission race. The lockfile function
uses an appropriate open call with mode argument so that the
lockfile is created atomically with both group ownership and
group write bit.
Bug: https://bugs.gentoo.org/468990
Signed-off-by: Zac Medico gentoo.org>
bin/ebuild-ipc.py| 6 +++---
bin/phase-functions.sh | 4 ++--
lib/_emerge/AbstractEbuildProcess.py | 4 ++--
lib/_emerge/EbuildIpcDaemon.py | 2 +-
lib/portage/package/ebuild/prepare_build_dirs.py | 9 +
lib/portage/tests/ebuild/test_doebuild_spawn.py | 1 +
lib/portage/tests/ebuild/test_ipc_daemon.py | 6 +++---
7 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py
index 4999c043a..c24ba6f58 100755
--- a/bin/ebuild-ipc.py
+++ b/bin/ebuild-ipc.py
@@ -138,9 +138,9 @@ class EbuildIpc:
def __init__(self):
self.fifo_dir = os.environ["PORTAGE_BUILDDIR"]
-self.ipc_in_fifo = os.path.join(self.fifo_dir, ".ipc_in")
-self.ipc_out_fifo = os.path.join(self.fifo_dir, ".ipc_out")
-self.ipc_lock_file = os.path.join(self.fifo_dir, ".ipc_lock")
+self.ipc_in_fifo = os.path.join(self.fifo_dir, ".ipc", "in")
+self.ipc_out_fifo = os.path.join(self.fifo_dir, ".ipc", "out")
+self.ipc_lock_file = os.path.join(self.fifo_dir, ".ipc", "lock")
def _daemon_is_alive(self):
try:
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index d3221993d..5eb031805 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -291,10 +291,10 @@ __dyn_clean() {
rm -f
"$PORTAGE_BUILDDIR"/.{ebuild_changed,logid,pretended,setuped,unpacked,prepared}
\
"$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged,instprepped} \
"$PORTAGE_BUILDDIR"/.die_hooks \
- "$PORTAGE_BUILDDIR"/.ipc_{in,out,lock} \
"$PORTAGE_BUILDDIR"/.exit_status
- rm -rf "${PORTAGE_BUILDDIR}/build-info"
+ rm -rf "${PORTAGE_BUILDDIR}/build-info" \
+ "${PORTAGE_BUILDDIR}/.ipc"
rm -rf "${WORKDIR}"
rm -f "${PORTAGE_BUILDDIR}/files"
fi
diff --git a/lib/_emerge/AbstractEbuildProcess.py
b/lib/_emerge/AbstractEbuildProcess.py
index 1b4e7759f..6d89d40f0 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -249,8 +249,8 @@ class AbstractEbuildProcess(SpawnProcess):
def _init_ipc_fifos(self):
-input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in")
-output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".ipc_out")
+input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc",
"in")
+output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc",
"out")
for p in (input_fifo, output_fifo):
diff --git a/lib/_emerge/EbuildIpcDaemon.py b/lib/_emerge/EbuildIpcDaemon.py
index ee6fd7658..78594ff0a 100644
--- a/lib/_emerge/EbuildIpcDaemon.py
+++ b/lib/_emerge/EbuildIpcDaemon.py
@@ -81,7 +81,7 @@ class EbuildIpcDaemon(FifoIpcDaemon):
# write something to the pipe just before we close it, and in that
# case the write will be lost. Therefore, try for a non-blocking
# lock, and only re-open the pipe if the lock is acquired.
-lock_filename = os.path.join(os.path.dirname(self.input_fifo),
".ipc_lock")
+lock_filename = os.path.join(os.path.dirname(self.input_fifo),
"lock")
try:
lock_obj = lockfile(lock_filename, unlinkfile=True,
flags=os.O_NONBLOCK)
except TryAgain:
diff --git a/lib/portage/package/ebuild/prepare_build_dirs.py
b/lib/portage/package/ebuild/prepare_build_dirs.py
index 659198905..32a770c99 100644
--- a/lib/portage/package/ebuild/prepare_build_dirs.py
+++ b/lib/portage/package/ebuild/prepare_build_dirs.py
@@ -102,6 +102,15 @@ def prepare_build_dirs(myroot=None, settings=None,
cleanup=False):
apply_secpass_permissions(
mysettings[dir_key], uid=portage_uid, gid=portage_gid
)
+# The setgid bit prevents a lockfile group permission race for bug
#468990.
+ipc_kwargs = {}
+if portage.data.secpass >= 1:
+ipc_kwargs["gid"] = portage_gid
+ipc_kwargs["mode"] = 0o2770
+ensure_dirs(
+os.path.join(mysettings["PORTAGE_BUILDDIR"], ".ipc"),
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
commit: 5d61d0f8d72b27213896f052ef4abb2337b922bf
Author: Zac Medico gentoo org>
AuthorDate: Tue Sep 21 04:53:52 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Sep 21 05:33:50 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5d61d0f8
Binpkg: convert compat coroutine to async
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/Binpkg.py | 10 --
lib/portage/util/_async/AsyncTaskFuture.py | 4 +++-
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index 0f37063f0..c7dde69bd 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2019 Gentoo Authors
+# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -15,7 +15,6 @@ from _emerge.SpawnProcess import SpawnProcess
from portage.eapi import eapi_exports_replace_vars
from portage.util import ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
-from portage.util.futures.compat_coroutine import coroutine
import portage
from portage import os
from portage import shutil
@@ -305,8 +304,7 @@ class Binpkg(CompositeTask):
self._unpack_metadata_exit,
)
-@coroutine
-def _unpack_metadata(self, loop=None):
+async def _unpack_metadata(self, loop=None):
dir_path = self.settings["PORTAGE_BUILDDIR"]
@@ -327,7 +325,7 @@ class Binpkg(CompositeTask):
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
-yield self._bintree.dbapi.unpack_metadata(
+await self._bintree.dbapi.unpack_metadata(
self.settings, infloc, loop=self.scheduler
)
check_missing_metadata = ("CATEGORY", "PF")
@@ -378,7 +376,7 @@ class Binpkg(CompositeTask):
background=self.background, scheduler=self.scheduler,
settings=self.settings
)
env_extractor.start()
-yield env_extractor.async_wait()
+await env_extractor.async_wait()
if env_extractor.returncode != os.EX_OK:
raise portage.exception.PortageException(
"failed to extract environment for {}".format(self.pkg.cpv)
diff --git a/lib/portage/util/_async/AsyncTaskFuture.py
b/lib/portage/util/_async/AsyncTaskFuture.py
index f87a7f90a..0cd034c97 100644
--- a/lib/portage/util/_async/AsyncTaskFuture.py
+++ b/lib/portage/util/_async/AsyncTaskFuture.py
@@ -1,10 +1,11 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2021 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import os
import signal
from _emerge.AsynchronousTask import AsynchronousTask
+from portage.util.futures import asyncio
class AsyncTaskFuture(AsynchronousTask):
@@ -16,6 +17,7 @@ class AsyncTaskFuture(AsynchronousTask):
__slots__ = ("future",)
def _start(self):
+self.future = asyncio.ensure_future(self.future, self.scheduler)
self.future.add_done_callback(self._done_callback)
def _cancel(self):
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/emerge/
commit: 971546efe69394743584107c0657b62fb65ee163
Author: Zac Medico gentoo org>
AuthorDate: Wed Apr 21 04:28:12 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Apr 21 04:29:21 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=971546ef
emerge: use parse_intermixed_args when available (bug 784566)
The included unit test case previously failed with this error:
emerge: error: unrecognized arguments: dev-libs/A
Bug: https://bugs.gentoo.org/784566
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/main.py | 2 +-
lib/portage/tests/emerge/test_simple.py | 10 +-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 31e690584..191be6479 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -834,7 +834,7 @@ def parse_opts(tmpcmdline, silent=False):
tmpcmdline = insert_optional_args(tmpcmdline)
- myoptions = parser.parse_args(args=tmpcmdline)
+ myoptions = getattr(parser, "parse_intermixed_args",
parser.parse_args)(args=tmpcmdline)
if myoptions.alert in true_y:
myoptions.alert = True
diff --git a/lib/portage/tests/emerge/test_simple.py
b/lib/portage/tests/emerge/test_simple.py
index 6e282337f..5b110407f 100644
--- a/lib/portage/tests/emerge/test_simple.py
+++ b/lib/portage/tests/emerge/test_simple.py
@@ -1,6 +1,7 @@
# Copyright 2011-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import argparse
import subprocess
import portage
@@ -289,7 +290,14 @@ call_has_and_best_version() {
port=binhost_server.server_port,
path=binhost_remote_path)
- test_commands = (
+ test_commands = ()
+
+ if hasattr(argparse.ArgumentParser, "parse_intermixed_args"):
+ test_commands += (
+ emerge_cmd + ("--oneshot", "dev-libs/A", "-v",
"dev-libs/A"),
+ )
+
+ test_commands += (
emerge_cmd + ("--usepkgonly", "--root", cross_root,
"--quickpkg-direct=y", "--quickpkg-direct-root", "/", "dev-libs/A"),
emerge_cmd + ("--usepkgonly", "--quickpkg-direct=y",
"--quickpkg-direct-root", cross_root, "dev-libs/A"),
env_update_cmd,
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
commit: 77e545e0955930606eb7e73d3970d8c0706fd8a8
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 7 14:55:49 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 7 15:12:42 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=77e545e0
EbuildPhase: Use async and await syntax
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildPhase.py| 28 ---
lib/portage/util/_async/SchedulerInterface.py | 10 --
2 files changed, 16 insertions(+), 22 deletions(-)
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index e4c0428a6..26c770d29 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -21,7 +21,6 @@ from portage.util._dyn_libs.soname_deps_qa import (
)
from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir,
_prepare_fake_distdir, _prepare_fake_filesdir)
-from portage.util.futures.compat_coroutine import coroutine
from portage.util import writemsg
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -70,11 +69,10 @@ class EbuildPhase(CompositeTask):
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
def _start(self):
- future =
asyncio.ensure_future(self._async_start(loop=self.scheduler),
loop=self.scheduler)
+ future = asyncio.ensure_future(self._async_start(),
loop=self.scheduler)
self._start_task(AsyncTaskFuture(future=future),
self._async_start_exit)
- @coroutine
- def _async_start(self, loop=None):
+ async def _async_start(self):
need_builddir = self.phase not in
EbuildProcess._phases_without_builddir
@@ -132,7 +130,7 @@ class EbuildPhase(CompositeTask):
# Force background=True for this header since it's
intended
# for the log and it doesn't necessarily need to be
visible
# elsewhere.
- yield self._elog('einfo', msg, background=True,
loop=self.scheduler)
+ await self._elog('einfo', msg, background=True)
if self.phase == 'package':
if 'PORTAGE_BINPKG_TMPFILE' not in self.settings:
@@ -402,8 +400,7 @@ class EbuildPhase(CompositeTask):
self.returncode = 1
self.wait()
- @coroutine
- def _elog(self, elog_funcname, lines, background=None, loop=None):
+ async def _elog(self, elog_funcname, lines, background=None):
if background is None:
background = self.background
out = io.StringIO()
@@ -434,12 +431,12 @@ class EbuildPhase(CompositeTask):
_set_nonblocking(build_logger.stdin.fileno())
log_file = build_logger.stdin
- yield self.scheduler.async_output(msg,
log_file=log_file,
- background=background,
loop=self.scheduler)
+ await self.scheduler.async_output(msg,
log_file=log_file,
+ background=background)
if build_logger is not None:
build_logger.stdin.close()
- yield build_logger.async_wait()
+ await build_logger.async_wait()
except asyncio.CancelledError:
if build_logger is not None:
build_logger.cancel()
@@ -487,7 +484,7 @@ class _PostPhaseCommands(CompositeTask):
if 'qa-unresolved-soname-deps' in
self.settings.features:
# This operates on REQUIRES metadata generated
by the above function call.
- future =
asyncio.ensure_future(self._soname_deps_qa(loop=self.scheduler),
loop=self.scheduler)
+ future =
asyncio.ensure_future(self._soname_deps_qa(), loop=self.scheduler)
# If an unexpected exception occurs, then this
will raise it.
future.add_done_callback(lambda future:
future.cancelled() or future.result())
self._start_task(AsyncTaskFuture(future=future), self._default_final_exit)
@@ -496,12 +493,11 @@ class _PostPhaseCommands(CompositeTask):
else:
self._default_final_exit(task)
- @coroutine
- def _soname_deps_qa(self, loop=None):
+ async def _soname_deps_qa(self):
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 5095c2023595a75e2848f1ad3dbe25b5fb451a44 Author: Zac Medico gentoo org> AuthorDate: Mon Nov 16 05:55:54 2020 + Commit: Zac Medico gentoo org> CommitDate: Sun Nov 22 03:19:29 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5095c202 find_smallest_cycle: enhance search prioritization Enhance the find_smallest_cycle function to prioritize its search so that it will minimize the use of installed packages to break cycles. When installed packages must be used to break cycles, it will now prefer to do this for runtime dependencies over buildtime dependencies, since it's preferable to build against latest versions of buildtime dependencies whenever possible. This should solve some cases of bug 199856 which have been triggered by unsafe reliance on installed packages to break cycles. The included unit test case demonstrates correct merge order for a dependency calculation involving 6 independent cycles. This test case fails with the master branch, due to a buildtime dependency cycle of 3 packages being merged earlier than cycles of 2 packages. We can generalize this to say that the master branch may use an installed package to break an arbitrarily sized cycle in a somewhat random location, even though that cycle may be composed of smaller independent cycles which would be safer to break individually. Bug: https://bugs.gentoo.org/754903 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/DepPriorityNormalRange.py | 2 + lib/_emerge/DepPrioritySatisfiedRange.py | 53 -- lib/_emerge/depgraph.py| 43 - lib/portage/tests/resolver/test_merge_order.py | 10 + 4 files changed, 70 insertions(+), 38 deletions(-) diff --git a/lib/_emerge/DepPriorityNormalRange.py b/lib/_emerge/DepPriorityNormalRange.py index 5f3f3da70..10f205a3b 100644 --- a/lib/_emerge/DepPriorityNormalRange.py +++ b/lib/_emerge/DepPriorityNormalRange.py @@ -14,6 +14,7 @@ class DepPriorityNormalRange: """ MEDIUM = 3 MEDIUM_SOFT = 2 + MEDIUM_POST = 2 SOFT= 1 NONE= 0 @@ -37,6 +38,7 @@ class DepPriorityNormalRange: ignore_medium = _ignore_runtime ignore_medium_soft = _ignore_runtime_post + ignore_medium_post = _ignore_runtime_post ignore_soft= _ignore_optional DepPriorityNormalRange.ignore_priority = ( diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py b/lib/_emerge/DepPrioritySatisfiedRange.py index e056e676f..fb0d7db4e 100644 --- a/lib/_emerge/DepPrioritySatisfiedRange.py +++ b/lib/_emerge/DepPrioritySatisfiedRange.py @@ -8,17 +8,18 @@ class DepPrioritySatisfiedRange: not satisfied and buildtimeHARD not satisfied and runtime 7 MEDIUM - not satisfied and runtime_post 6 MEDIUM_SOFT - satisfied and buildtime_slot_op5 SOFT - satisfied and buildtime4 SOFT - satisfied and runtime 3 SOFT - satisfied and runtime_post 2 SOFT + satisfied and buildtime_slot_op6 MEDIUM_SOFT + satisfied and buildtime5 MEDIUM_SOFT + satisfied and runtime 4 MEDIUM_SOFT + runtime_post 3 MEDIUM_POST + satisfied and runtime_post 2 MEDIUM_POST optional 1 SOFT (none of the above)0 NONE """ MEDIUM = 7 MEDIUM_SOFT = 6 - SOFT= 5 + MEDIUM_POST = 3 + SOFT= 1 NONE= 0 @classmethod @@ -35,42 +36,51 @@ class DepPrioritySatisfiedRange: return True if not priority.satisfied: return False + if priority.buildtime or priority.runtime: + return False return bool(priority.runtime_post) @classmethod - def _ignore_satisfied_runtime(cls, priority): + def _ignore_runtime_post(cls, priority): if priority.__class__ is not DepPriority: return False if priority.optional: return True - if not priority.satisfied: + if priority.buildtime or priority.runtime: return False - return not priority.buildtime + return bool(priority.runtime_post) @classmethod - def _ignore_satisfied_buildtime(cls, priority): + def _ignore_satisfied_runtime(cls, priority): if priority.__class__ is not DepPriority: return False if priority.optional: return True - if priority.buildtime_slot_op: +
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: f64310749f176f8921b72ce282b4294efe81c3f0
Author: Zac Medico gentoo org>
AuthorDate: Sat Sep 19 21:32:41 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Sep 20 22:27:32 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f6431074
_slot_confict_backtrack: minimize conflict atoms (bug 743631)
Prefer choices that minimize conflict atoms, so that choices
which satisfy all parents are preferred. This reduces the
minimum necessary backtrack tries from 21 to 7 for the unit
test related to bug 743115.
Bug: https://bugs.gentoo.org/743115
Bug: https://bugs.gentoo.org/743631
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py| 6 ++
lib/portage/tests/resolver/test_slot_operator_missed_update.py | 2 +-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 3f864aefc..7281d8692 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1797,6 +1797,12 @@ class depgraph:
if parent_atom not in parent_atoms)
backtrack_data.append((to_be_masked, conflict_atoms))
+ # Prefer choices that minimize conflict atoms. This is intended
+ # to take precedence over the earlier package version sort. The
+ # package version sort is still needed or else choices for the
+ # testOverlapSlotConflict method of
VirtualMinimizeChildrenTestCase
+ # become non-deterministic.
+ backtrack_data.sort(key=lambda item: len(item[1]))
to_be_masked = backtrack_data[-1][0]
self._dynamic_config._backtrack_infos.setdefault(
diff --git a/lib/portage/tests/resolver/test_slot_operator_missed_update.py
b/lib/portage/tests/resolver/test_slot_operator_missed_update.py
index fce012f62..1ea701003 100644
--- a/lib/portage/tests/resolver/test_slot_operator_missed_update.py
+++ b/lib/portage/tests/resolver/test_slot_operator_missed_update.py
@@ -90,7 +90,7 @@ class BacktrackMissedUpdateTestCase(TestCase):
# Bug 743115: missed updates trigger excessive
backtracking
ResolverPlaygroundTestCase(
[">=dev-python/pypy3-7.3.2_rc", "@world"],
- options={"--update": True, "--deep": True,
"--backtrack": 25},
+ options={"--update": True, "--deep": True,
"--backtrack": 10},
success=True,
mergelist=[
"dev-python/pypy3-7.3.2_rc2_p37-r1",
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/sync/modules/rsync/, lib/portage/util/_async/, ...
commit: 762f5aff42f9240cb732993b5ba4e99fa2b9383b Author: Aaron Bauman gentoo org> AuthorDate: Thu Aug 6 18:21:42 2020 + Commit: Zac Medico gentoo org> CommitDate: Fri Aug 7 02:53:11 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=762f5aff lib/*: Fix ungrouped-imports * This fixes ungrouped imports for a portion of the repo. The remaining fixes will come in another batch of atomic commits as the imports are buried behind conditionals, are __futures__ that may be able to go away, etc * Per the above, check will not be enabled yet Signed-off-by: Aaron Bauman gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/_emerge/FakeVartree.py | 7 --- lib/_emerge/FifoIpcDaemon.py| 2 +- lib/_emerge/MetadataRegen.py| 7 --- lib/portage/dbapi/porttree.py | 2 +- lib/portage/emaint/modules/move/move.py | 5 +++-- lib/portage/metadata.py | 16 +--- lib/portage/sync/modules/rsync/rsync.py | 29 +++-- lib/portage/util/_async/BuildLogger.py | 4 ++-- 8 files changed, 39 insertions(+), 33 deletions(-) diff --git a/lib/_emerge/FakeVartree.py b/lib/_emerge/FakeVartree.py index 37349bcdd..c5c9e27ca 100644 --- a/lib/_emerge/FakeVartree.py +++ b/lib/_emerge/FakeVartree.py @@ -3,10 +3,12 @@ import warnings -import portage -from portage import os from _emerge.Package import Package from _emerge.PackageVirtualDbapi import PackageVirtualDbapi +from _emerge.resolver.DbapiProvidesIndex import PackageDbapiProvidesIndex + +import portage +from portage import os from portage.const import VDB_PATH from portage.dbapi.vartree import vartree from portage.dep._slot_operator import find_built_slot_operator_atoms @@ -14,7 +16,6 @@ from portage.eapi import _get_eapi_attrs from portage.exception import InvalidData, InvalidDependString from portage.update import grab_updates, parse_updates, update_dbentries from portage.versions import _pkg_str -from _emerge.resolver.DbapiProvidesIndex import PackageDbapiProvidesIndex class FakeVardbGetPath: diff --git a/lib/_emerge/FifoIpcDaemon.py b/lib/_emerge/FifoIpcDaemon.py index a21e94db6..22aa6f9fc 100644 --- a/lib/_emerge/FifoIpcDaemon.py +++ b/lib/_emerge/FifoIpcDaemon.py @@ -1,9 +1,9 @@ # Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +from _emerge.AbstractPollTask import AbstractPollTask from portage import os -from _emerge.AbstractPollTask import AbstractPollTask from portage.cache.mappings import slot_dict_class class FifoIpcDaemon(AbstractPollTask): diff --git a/lib/_emerge/MetadataRegen.py b/lib/_emerge/MetadataRegen.py index 8eb110a46..967554503 100644 --- a/lib/_emerge/MetadataRegen.py +++ b/lib/_emerge/MetadataRegen.py @@ -1,11 +1,12 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +from _emerge.EbuildMetadataPhase import EbuildMetadataPhase + import portage from portage import os -from portage.dep import _repo_separator -from _emerge.EbuildMetadataPhase import EbuildMetadataPhase from portage.cache.cache_errors import CacheError +from portage.dep import _repo_separator from portage.util._async.AsyncScheduler import AsyncScheduler class MetadataRegen(AsyncScheduler): diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 9ad8addc7..0c2a4e2b3 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -42,9 +42,9 @@ import os as _os import traceback import warnings import errno -import collections import functools +import collections from collections import OrderedDict from urllib.parse import urlparse diff --git a/lib/portage/emaint/modules/move/move.py b/lib/portage/emaint/modules/move/move.py index e2df53fd8..8fc3269ca 100644 --- a/lib/portage/emaint/modules/move/move.py +++ b/lib/portage/emaint/modules/move/move.py @@ -1,10 +1,11 @@ -# Copyright 2005-2018 Gentoo Foundation +# Copyright 2005-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +from _emerge.Package import Package + import portage from portage import os from portage.exception import InvalidData -from _emerge.Package import Package from portage.versions import _pkg_str class MoveHandler: diff --git a/lib/portage/metadata.py b/lib/portage/metadata.py index 935bcc307..f932c094a 100644 --- a/lib/portage/metadata.py +++ b/lib/portage/metadata.py @@ -1,19 +1,21 @@ +# Copyright 1998-2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 from __future__ import print_function -import sys -import signal import logging import operator - import portage -from portage import os +import sys +import signal + +from _emerge.ProgressHandler import ProgressHandler + from portage import eapi_is_supported -from portage.util import
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/_parallel_manifest/, ...
commit: 709826c0896120e8b3d9995d71678c0b9b290e4c Author: Alec Warner gentoo org> AuthorDate: Wed Jul 29 18:27:16 2020 + Commit: Zac Medico gentoo org> CommitDate: Thu Jul 30 04:05:06 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=709826c0 Fix R1705. The codebase appears to have this pattern like: if foo == bar: return a elif foo == baz: return b else: return c This can often be rewritten as: if foo == bar: return a if foo == baz: return b return c Closes: https://github.com/gentoo/portage/pull/592 Signed-off-by: Alec Warner gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/_emerge/EbuildBuild.py | 24 ++-- lib/_emerge/MergeListItem.py | 2 +- lib/_emerge/Package.py | 3 +- lib/_emerge/actions.py | 11 +- lib/_emerge/depgraph.py| 131 ++--- lib/_emerge/main.py| 4 +- lib/_emerge/resolver/backtracking.py | 3 +- lib/_emerge/resolver/output.py | 34 +++--- lib/_emerge/resolver/slot_collision.py | 32 +++-- lib/_emerge/unmerge.py | 4 +- lib/portage/_emirrordist/FetchTask.py | 36 +++--- lib/portage/_global_updates.py | 3 +- lib/portage/_legacy_globals.py | 2 +- lib/portage/_sets/__init__.py | 7 +- lib/portage/_sets/base.py | 3 +- lib/portage/_sets/dbapi.py | 43 --- lib/portage/cache/ebuild_xattr.py | 3 +- lib/portage/cache/mappings.py | 4 +- lib/portage/cache/sqlite.py| 5 +- lib/portage/checksum.py| 2 +- lib/portage/cvstree.py | 5 +- lib/portage/data.py| 5 +- lib/portage/dbapi/IndexedPortdb.py | 3 +- lib/portage/dbapi/IndexedVardb.py | 3 +- lib/portage/dbapi/bintree.py | 14 +-- lib/portage/dbapi/cpv_expand.py| 6 +- lib/portage/dbapi/porttree.py | 5 +- lib/portage/dbapi/vartree.py | 6 +- lib/portage/dbapi/virtual.py | 3 +- lib/portage/debug.py | 12 +- lib/portage/dep/__init__.py| 22 ++-- lib/portage/dep/_dnf.py| 2 +- lib/portage/dep/dep_check.py | 13 +- lib/portage/elog/mod_mail_summary.py | 2 +- lib/portage/exception.py | 3 +- lib/portage/getbinpkg.py | 8 +- lib/portage/glsa.py| 5 +- lib/portage/locks.py | 15 ++- lib/portage/manifest.py| 7 +- lib/portage/output.py | 30 +++-- lib/portage/package/ebuild/_ipc/QueryCommand.py| 15 ++- .../ebuild/_parallel_manifest/ManifestProcess.py | 3 +- lib/portage/package/ebuild/config.py | 17 ++- lib/portage/package/ebuild/doebuild.py | 8 +- lib/portage/package/ebuild/fetch.py| 10 +- lib/portage/package/ebuild/getmaskingreason.py | 9 +- lib/portage/package/ebuild/getmaskingstatus.py | 2 +- lib/portage/repository/config.py | 3 +- lib/portage/sync/controller.py | 3 +- lib/portage/sync/modules/git/git.py| 33 +++--- lib/portage/tests/util/futures/test_retry.py | 12 +- lib/portage/util/__init__.py | 33 ++ lib/portage/util/_async/PipeLogger.py | 89 +++--- lib/portage/util/_dyn_libs/LinkageMapELF.py| 33 +++--- lib/portage/util/_urlopen.py | 42 +++ lib/portage/util/changelog.py | 19 ++- lib/portage/util/futures/_asyncio/__init__.py | 5 +- lib/portage/util/futures/unix_events.py| 9 +- lib/portage/util/lafilefixer.py| 3 +- lib/portage/versions.py| 17 ++- 60 files changed, 409 insertions(+), 481 deletions(-) diff --git a/lib/_emerge/EbuildBuild.py b/lib/_emerge/EbuildBuild.py index a575835b7..4da815988 100644 --- a/lib/_emerge/EbuildBuild.py +++ b/lib/_emerge/EbuildBuild.py @@ -159,18 +159,18 @@ class EbuildBuild(CompositeTask): settings=self.settings), self._default_final_exit) return - else: - fetcher = EbuildFetcher( -
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/futures/executor/
commit: 174b624d6a2d863d9779b0c2c4035ff33ba46a90 Author: Zac Medico gentoo org> AuthorDate: Wed Apr 8 05:01:31 2020 + Commit: Zac Medico gentoo org> CommitDate: Wed Apr 8 05:29:48 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=174b624d Revert "ForkExecutor: use async_start method" This reverts commit 4df7a0a0c16c5ded65ad601d39840797b7704770. Bug: https://bugs.gentoo.org/716636 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/AsynchronousTask.py | 15 ++- lib/portage/util/futures/executor/fork.py | 21 +++-- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/lib/_emerge/AsynchronousTask.py b/lib/_emerge/AsynchronousTask.py index c4197d468..962c68547 100644 --- a/lib/_emerge/AsynchronousTask.py +++ b/lib/_emerge/AsynchronousTask.py @@ -25,19 +25,8 @@ class AsynchronousTask(SlotObject): @coroutine def async_start(self): - try: - if self._was_cancelled(): - raise asyncio.CancelledError - yield self._async_start() - if self._was_cancelled(): - raise asyncio.CancelledError - except asyncio.CancelledError: - self.cancel() - self._was_cancelled() - self._async_wait() - raise - finally: - self._start_hook() + yield self._async_start() + self._start_hook() @coroutine def _async_start(self): diff --git a/lib/portage/util/futures/executor/fork.py b/lib/portage/util/futures/executor/fork.py index 3549fdb31..add7b3c9e 100644 --- a/lib/portage/util/futures/executor/fork.py +++ b/lib/portage/util/futures/executor/fork.py @@ -13,7 +13,6 @@ import traceback from portage.util._async.AsyncFunction import AsyncFunction from portage.util.futures import asyncio -from portage.util.futures.compat_coroutine import coroutine from portage.util.cpuinfo import get_cpu_count @@ -52,25 +51,11 @@ class ForkExecutor(object): while (not self._shutdown and self._submit_queue and len(self._running_tasks) < self._max_workers): future, proc = self._submit_queue.popleft() + future.add_done_callback(functools.partial(self._cancel_cb, proc)) + proc.addExitListener(functools.partial(self._proc_exit, future)) proc.scheduler = self._loop + proc.start() self._running_tasks[id(proc)] = proc - future.add_done_callback(functools.partial(self._cancel_cb, proc)) - proc_future = asyncio.ensure_future(self._proc_coroutine(proc), loop=self._loop) - proc_future.add_done_callback(functools.partial(self._proc_coroutine_done, future, proc)) - - @coroutine - def _proc_coroutine(self, proc): - yield proc.async_start() - yield proc.async_wait() - - def _proc_coroutine_done(self, future, proc, proc_future): - try: - proc_future.result() - except asyncio.CancelledError: - future.done() or future.cancel() - if proc.poll() is None: - proc.cancel() - self._proc_exit(future, proc) def _cancel_cb(self, proc, future): if future.cancelled():
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/, lib/portage/package/ebuild/_config/, ...
commit: 17df30bdfb72a157f9e34f3c2efa1e59389861d4
Author: Zac Medico gentoo org>
AuthorDate: Wed Apr 8 04:34:56 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Apr 8 05:29:45 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=17df30bd
Revert "Rename PORTAGE_LOG_FILTER_FILE_CMD from PORTAGE_LOG_FILTER_FILE"
This reverts commit e24859eaa03ec86e10d842296f5570dd98bed4b7.
Bug: https://bugs.gentoo.org/716636
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/AbstractEbuildProcess.py | 2 +-
lib/_emerge/EbuildPhase.py | 2 +-
lib/portage/package/ebuild/_config/special_env_vars.py | 4 ++--
lib/portage/util/_async/BuildLogger.py | 2 +-
man/make.conf.5| 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/lib/_emerge/AbstractEbuildProcess.py
b/lib/_emerge/AbstractEbuildProcess.py
index cc1ea0f1a..6b2c2f81b 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -181,7 +181,7 @@ class AbstractEbuildProcess(SpawnProcess):
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
- self.log_filter_file =
self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
+ self.log_filter_file =
self.settings.get('PORTAGE_LOG_FILTER_FILE')
try:
yield SpawnProcess._async_start(self)
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index fbb040ffd..927a74b98 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -420,7 +420,7 @@ class EbuildPhase(CompositeTask):
if log_path:
build_logger =
BuildLogger(env=self.settings.environ(),
log_path=log_path,
-
log_filter_file=self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD'),
+
log_filter_file=self.settings.get('PORTAGE_LOG_FILTER_FILE'),
scheduler=self.scheduler)
yield build_logger.async_start()
log_file = build_logger.stdin
diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py
b/lib/portage/package/ebuild/_config/special_env_vars.py
index c6f88f08c..dd8105123 100644
--- a/lib/portage/package/ebuild/_config/special_env_vars.py
+++ b/lib/portage/package/ebuild/_config/special_env_vars.py
@@ -175,7 +175,7 @@ environ_filter += [
"PORTAGE_RO_DISTDIRS",
"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
"PORTAGE_RSYNC_RETRIES", "PORTAGE_SSH_OPTS", "PORTAGE_SYNC_STALE",
- "PORTAGE_USE", "PORTAGE_LOG_FILTER_FILE_CMD",
+ "PORTAGE_USE", "PORTAGE_LOG_FILTER_FILE",
"PORTAGE_LOGDIR", "PORTAGE_LOGDIR_CLEAN",
"QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS",
"RESUMECOMMAND", "RESUMECOMMAND_FTP",
@@ -205,7 +205,7 @@ default_globals = {
}
validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND',
- 'PORTAGE_LOG_FILTER_FILE_CMD',
+ 'PORTAGE_LOG_FILTER_FILE',
)
# To enhance usability, make some vars case insensitive
diff --git a/lib/portage/util/_async/BuildLogger.py
b/lib/portage/util/_async/BuildLogger.py
index 49f1321fb..4873d9750 100644
--- a/lib/portage/util/_async/BuildLogger.py
+++ b/lib/portage/util/_async/BuildLogger.py
@@ -14,7 +14,7 @@ class BuildLogger(AsynchronousTask):
Write to a log file, with compression support provided by PipeLogger.
If the log_filter_file parameter is specified, then it is interpreted
as a command to execute which filters log output (see the
- PORTAGE_LOG_FILTER_FILE_CMD variable in make.conf(5)). The stdin
property
+ PORTAGE_LOG_FILTER_FILE variable in make.conf(5)). The stdin property
provides access to a writable binary file stream (refers to a pipe)
that log content should be written to (usually redirected from
subprocess stdout and stderr streams).
diff --git a/man/make.conf.5 b/man/make.conf.5
index 467a9d394..baecd283a 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -979,7 +979,7 @@ with an integer pid. For example, a value of "ionice \-c 3
\-p \\${PID}"
will set idle io priority. For more information about ionice, see
\fBionice\fR(1). This variable is unset by default.
.TP
-.B PORTAGE_LOG_FILTER_FILE_CMD
+.B PORTAGE_LOG_FILTER_FILE
This variable specifies a command that filters build log output to a
log file. In order to filter ANSI escape codes from build logs,
\fBansifilter\fR(1) is a convenient setting for this variable.
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/, lib/portage/tests/util/futures/, ...
commit: 28adac2b67f46e11b4cf9116761d888e91e55f3c
Author: Zac Medico gentoo org>
AuthorDate: Wed Apr 8 05:00:56 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Apr 8 05:29:48 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=28adac2b
Revert "SpawnProcess: use async_start method (bug 709746)"
This reverts commit 5c40c3e7ec180c9c7d1eea521d69487177c7f519.
Bug: https://bugs.gentoo.org/716636
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/AbstractEbuildProcess.py | 2 +-
lib/_emerge/AsynchronousLock.py | 15 +++
lib/_emerge/BinpkgExtractorAsync.py | 9 ++---
lib/_emerge/BinpkgFetcher.py | 9 ++---
lib/_emerge/EbuildFetcher.py | 9 ++---
lib/_emerge/SpawnProcess.py | 8 ++--
lib/portage/dbapi/bintree.py | 4 ++--
lib/portage/tests/util/futures/test_iter_completed.py | 6 +-
lib/portage/util/_async/AsyncFunction.py | 9 ++---
lib/portage/util/_async/FileDigester.py | 9 ++---
10 files changed, 19 insertions(+), 61 deletions(-)
diff --git a/lib/_emerge/AbstractEbuildProcess.py
b/lib/_emerge/AbstractEbuildProcess.py
index 09b76830d..fd8a40cc1 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -182,7 +182,7 @@ class AbstractEbuildProcess(SpawnProcess):
self.fd_pipes[0] = null_fd
try:
- yield SpawnProcess._async_start(self)
+ SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
diff --git a/lib/_emerge/AsynchronousLock.py b/lib/_emerge/AsynchronousLock.py
index 9efaaceac..aed1bcb15 100644
--- a/lib/_emerge/AsynchronousLock.py
+++ b/lib/_emerge/AsynchronousLock.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import fcntl
@@ -21,7 +21,6 @@ from portage.exception import TryAgain
from portage.localization import _
from portage.locks import lockfile, unlockfile
from portage.util import writemsg_level
-from portage.util.futures.compat_coroutine import coroutine
from _emerge.AbstractPollTask import AbstractPollTask
from _emerge.AsynchronousTask import AsynchronousTask
from _emerge.SpawnProcess import SpawnProcess
@@ -44,10 +43,6 @@ class AsynchronousLock(AsynchronousTask):
_use_process_by_default = True
def _start(self):
- self.scheduler.run_until_complete(self._async_start())
-
- @coroutine
- def _async_start(self):
if not self._force_async:
try:
@@ -70,7 +65,7 @@ class AsynchronousLock(AsynchronousTask):
_force_dummy=self._force_dummy)
self._imp.addExitListener(self._imp_exit)
- yield self._imp.async_start()
+ self._imp.start()
def _imp_exit(self, imp):
# call exit listeners
@@ -188,10 +183,6 @@ class _LockProcess(AbstractPollTask):
('_acquired', '_kill_test', '_proc', '_files', '_unlock_future')
def _start(self):
- self.scheduler.run_until_complete(self._async_start())
-
- @coroutine
- def _async_start(self):
in_pr, in_pw = os.pipe()
out_pr, out_pw = os.pipe()
self._files = {}
@@ -220,7 +211,7 @@ class _LockProcess(AbstractPollTask):
fd_pipes={0:out_pr, 1:in_pw,
2:sys.__stderr__.fileno()},
scheduler=self.scheduler)
self._proc.addExitListener(self._proc_exit)
- yield self._proc.async_start()
+ self._proc.start()
os.close(out_pr)
os.close(in_pw)
diff --git a/lib/_emerge/BinpkgExtractorAsync.py
b/lib/_emerge/BinpkgExtractorAsync.py
index 5f4caa794..3733bdeb5 100644
--- a/lib/_emerge/BinpkgExtractorAsync.py
+++ b/lib/_emerge/BinpkgExtractorAsync.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import logging
@@ -10,7 +10,6 @@ from portage.util.compression_probe import (
compression_probe,
_compressors,
)
-from portage.util.futures.compat_coroutine import coroutine
from portage.process import find_binary
from portage.util import (
shlex_split,
@@ -28,10 +27,6 @@ class BinpkgExtractorAsync(SpawnProcess):
_shell_binary = portage.const.BASH_BINARY
def _start(self):
- self.scheduler.run_until_complete(self._async_start())
-
- @coroutine
- def _async_start(self):
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/util/futures/
commit: c1b19c2415c2f85074634da0c29350d905188c6f Author: Zac Medico gentoo org> AuthorDate: Sat Mar 7 20:25:40 2020 + Commit: Zac Medico gentoo org> CommitDate: Sat Mar 7 20:28:50 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c1b19c24 AsynchronousTask: handle addStartListener after exit (bug 711322) When addStartListener is called after the task has already exited with a returncode, immediately schedule the listener to be invoked via call_soon. This behavior is similar to the Future add_done_callback method (and addExitListener since commit 5d476c4e5002). Signed-off-by: Zac Medico gentoo.org> lib/_emerge/AsynchronousTask.py | 8 lib/portage/tests/util/futures/test_done_callback_after_exit.py | 4 2 files changed, 12 insertions(+) diff --git a/lib/_emerge/AsynchronousTask.py b/lib/_emerge/AsynchronousTask.py index 97db02587..ec5497b69 100644 --- a/lib/_emerge/AsynchronousTask.py +++ b/lib/_emerge/AsynchronousTask.py @@ -156,6 +156,10 @@ class AsynchronousTask(SlotObject): self._start_listeners = [] self._start_listeners.append(f) + # Ensure that start listeners are always called. + if self.returncode is not None: + self._start_hook() + def removeStartListener(self, f): if self._start_listeners is None: return @@ -198,6 +202,10 @@ class AsynchronousTask(SlotObject): used to trigger exit listeners when the returncode first becomes available. """ + # Ensure that start listeners are always called. + if self.returncode is not None: + self._start_hook() + if self.returncode is not None and \ self._exit_listeners is not None: diff --git a/lib/portage/tests/util/futures/test_done_callback_after_exit.py b/lib/portage/tests/util/futures/test_done_callback_after_exit.py index 46a51c271..8913d70e5 100644 --- a/lib/portage/tests/util/futures/test_done_callback_after_exit.py +++ b/lib/portage/tests/util/futures/test_done_callback_after_exit.py @@ -35,6 +35,10 @@ class DoneCallbackAfterExitTestCase(TestCase): loop.run_until_complete(task.async_wait()) for i in range(3): + event = loop.create_future() + task.addStartListener(lambda task: event.set_result(None)) + loop.run_until_complete(event) + event = loop.create_future() task.addExitListener(lambda task: event.set_result(None)) loop.run_until_complete(event)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/futures/executor/
commit: 4df7a0a0c16c5ded65ad601d39840797b7704770 Author: Zac Medico gentoo org> AuthorDate: Sun Feb 23 21:44:58 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Feb 24 02:35:15 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4df7a0a0 ForkExecutor: use async_start method Also, fix AsynchronousTask.async_start to handle cancellation of the _async_start coroutine, ensuring that start and exit listeners are notified in this case (otherwise RetryForkExecutorTestCase will hang). Signed-off-by: Zac Medico gentoo.org> lib/_emerge/AsynchronousTask.py | 15 +-- lib/portage/util/futures/executor/fork.py | 21 ++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/lib/_emerge/AsynchronousTask.py b/lib/_emerge/AsynchronousTask.py index d1e23cdf1..1e9e177cb 100644 --- a/lib/_emerge/AsynchronousTask.py +++ b/lib/_emerge/AsynchronousTask.py @@ -25,8 +25,19 @@ class AsynchronousTask(SlotObject): @coroutine def async_start(self): - yield self._async_start() - self._start_hook() + try: + if self._was_cancelled(): + raise asyncio.CancelledError + yield self._async_start() + if self._was_cancelled(): + raise asyncio.CancelledError + except asyncio.CancelledError: + self.cancel() + self._was_cancelled() + self._async_wait() + raise + finally: + self._start_hook() @coroutine def _async_start(self): diff --git a/lib/portage/util/futures/executor/fork.py b/lib/portage/util/futures/executor/fork.py index add7b3c9e..3549fdb31 100644 --- a/lib/portage/util/futures/executor/fork.py +++ b/lib/portage/util/futures/executor/fork.py @@ -13,6 +13,7 @@ import traceback from portage.util._async.AsyncFunction import AsyncFunction from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine from portage.util.cpuinfo import get_cpu_count @@ -51,11 +52,25 @@ class ForkExecutor(object): while (not self._shutdown and self._submit_queue and len(self._running_tasks) < self._max_workers): future, proc = self._submit_queue.popleft() - future.add_done_callback(functools.partial(self._cancel_cb, proc)) - proc.addExitListener(functools.partial(self._proc_exit, future)) proc.scheduler = self._loop - proc.start() self._running_tasks[id(proc)] = proc + future.add_done_callback(functools.partial(self._cancel_cb, proc)) + proc_future = asyncio.ensure_future(self._proc_coroutine(proc), loop=self._loop) + proc_future.add_done_callback(functools.partial(self._proc_coroutine_done, future, proc)) + + @coroutine + def _proc_coroutine(self, proc): + yield proc.async_start() + yield proc.async_wait() + + def _proc_coroutine_done(self, future, proc, proc_future): + try: + proc_future.result() + except asyncio.CancelledError: + future.done() or future.cancel() + if proc.poll() is None: + proc.cancel() + self._proc_exit(future, proc) def _cancel_cb(self, proc, future): if future.cancelled():
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 079f8c4a36ccc2ef5e25e7a57cd0707640f82592
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 14 19:21:28 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 14 23:10:37 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=079f8c4a
depclean: ensure consistency with update actions (bug 649622)
Make depclean traverse dependencies in the same order as update
actions, in order to ensure consistency in decisions which are
dependent on the order of dependency evaluation due to inconsistent
use of || preferences in different packages.
In unit tests, update test_virtual_w3m_realistic to assert that
the order of graph traversal is deterministic and consistent
between update and removal (depclean) actions.
Bug: https://bugs.gentoo.org/649622
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/actions.py | 33 +++---
lib/_emerge/depgraph.py | 21 +--
lib/portage/tests/resolver/ResolverPlayground.py | 77 +++-
lib/portage/tests/resolver/test_or_choices.py| 12 +++-
4 files changed, 96 insertions(+), 47 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 31252af16..4bf9ce425 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1,8 +1,9 @@
-# Copyright 1999-2019 Gentoo Authors
+# Copyright 1999-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import division, print_function, unicode_literals
+import collections
import errno
import logging
import operator
@@ -741,7 +742,19 @@ def action_depclean(settings, trees, ldpath_mtimes,
return rval
+
def calc_depclean(settings, trees, ldpath_mtimes,
+ myopts, action, args_set, spinner):
+ result = _calc_depclean(settings, trees, ldpath_mtimes,
+ myopts, action, args_set, spinner)
+ return result.returncode, result.cleanlist, result.ordered,
result.req_pkg_count
+
+
+_depclean_result = collections.namedtuple('_depclean_result',
+ ('returncode', 'cleanlist', 'ordered', 'req_pkg_count', 'depgraph'))
+
+
+def _calc_depclean(settings, trees, ldpath_mtimes,
myopts, action, args_set, spinner):
allow_missing_deps = bool(args_set)
@@ -805,7 +818,7 @@ def calc_depclean(settings, trees, ldpath_mtimes,
writemsg_level(_("!!! Aborting due to set configuration "
"errors displayed above.\n"),
level=logging.ERROR, noiselevel=-1)
- return 1, [], False, 0
+ return _depclean_result(1, [], False, 0, None)
if action == "depclean":
emergelog(xterm_titles, " >>> depclean")
@@ -920,7 +933,7 @@ def calc_depclean(settings, trees, ldpath_mtimes,
resolver.display_problems()
if not success:
- return 1, [], False, 0
+ return _depclean_result(1, [], False, 0, resolver)
def unresolved_deps():
@@ -1020,7 +1033,7 @@ def calc_depclean(settings, trees, ldpath_mtimes,
return False
if unresolved_deps():
- return 1, [], False, 0
+ return _depclean_result(1, [], False, 0, resolver)
graph = resolver._dynamic_config.digraph.copy()
required_pkgs_total = 0
@@ -1321,7 +1334,7 @@ def calc_depclean(settings, trees, ldpath_mtimes,
runtime_slot_op=True),
root=pkg.root)):
resolver.display_problems()
- return 1, [], False, 0
+ return _depclean_result(1, [],
False, 0, resolver)
writemsg_level("\nCalculating dependencies ")
success = resolver._complete_graph(
@@ -1329,9 +1342,9 @@ def calc_depclean(settings, trees, ldpath_mtimes,
writemsg_level("\b\b... done!\n")
resolver.display_problems()
if not success:
- return 1, [], False, 0
+ return _depclean_result(1, [], False, 0,
resolver)
if unresolved_deps():
- return 1, [], False, 0
+ return _depclean_result(1, [], False, 0,
resolver)
graph = resolver._dynamic_config.digraph.copy()
required_pkgs_total = 0
@@ -1340,7 +1353,7 @@ def calc_depclean(settings, trees, ldpath_mtimes,
required_pkgs_total += 1
cleanlist = create_cleanlist()
if not cleanlist:
- return 0, [], False, required_pkgs_total
+ return _depclean_result(0, [], False,
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/dbapi/
commit: 44076b9432a1361a39515927de2b60baa2fbddb9
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 1 21:23:39 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 1 21:30:23 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=44076b94
cpv_expand: treat GLEP 81 acct-* categories like virtual (bug 691798)
This solves common name collisions introduced by GLEP 81 user and group
management packages.
TODO: Provide user configuration so that users an ignore name
collisions with selected categories like app-emacs or dev-haskell.
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py | 2 +-
lib/portage/dbapi/cpv_expand.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index cae1c4470..1eaf20035 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -4212,7 +4212,7 @@ class depgraph(object):
if len(expanded_atoms) > 1:
number_of_virtuals = 0
for expanded_atom in expanded_atoms:
- if
expanded_atom.cp.startswith("virtual/"):
+ if
expanded_atom.cp.startswith(("acct-group/", "acct-user/", "virtual/")):
number_of_virtuals += 1
else:
candidate =
expanded_atom
diff --git a/lib/portage/dbapi/cpv_expand.py b/lib/portage/dbapi/cpv_expand.py
index 70ee78245..ac2f6cc2e 100644
--- a/lib/portage/dbapi/cpv_expand.py
+++ b/lib/portage/dbapi/cpv_expand.py
@@ -72,9 +72,9 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
matches.append(x+"/"+myp)
if len(matches) > 1:
virtual_name_collision = False
- if len(matches) == 2:
+ if len(matches) > 1:
for x in matches:
- if not x.startswith("virtual/"):
+ if not x.startswith(("acct-group/",
"acct-user/", "virtual/")):
# Assume that the non-virtual
is desired. This helps
# avoid the ValueError for
invalid deps that come from
# installed packages (during
reverse blocker detection,
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 1e61c439143b12d079e1fc344bbc0c192a84cbe0
Author: Zac Medico gentoo org>
AuthorDate: Wed Sep 11 02:54:51 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Sep 12 01:31:21 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1e61c439
_add_dep: less aggressive backtracking (bug 693836)
In order to suppress the sort of aggressive backtracking that can
trigger undesirable downgrades as in bug 693836, do not backtrack
for an unsatisfied dependency if there's an available package in
the runtime package mask which was involved in a slot conflict and
satisfied all involved parent atoms. Instead, discard the current
depgraph in favor of other backtracking configurations that may
exist. This case would not have been encountered prior to the fix
for bug 692746 which enabled backtracking for the type of slot
conflict that is detected here.
Fixes: 994ac00aa764 ("_slot_confict_backtrack: consider masking a package
matched by all parent atoms (bug 692746)")
Bug: https://bugs.gentoo.org/693836
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py| 13
.../test_aggressive_backtrack_downgrade.py | 91 ++
2 files changed, 104 insertions(+)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 6be1b3ec7..51614fc14 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -2888,6 +2888,19 @@ class depgraph(object):
dep.atom.without_use if
dep.atom.package
else dep.atom,
onlydeps=dep.onlydeps)
if dep_pkg is None:
+
+ # In order to suppress the sort
of aggressive
+ # backtracking that can trigger
undesirable downgrades
+ # as in bug 693836, do not
backtrack if there's an
+ # available package which was
involved in a slot
+ # conflict and satisfied all
involved parent atoms.
+ for dep_pkg, reasons in
self._dynamic_config._runtime_pkg_mask.items():
+ if
(dep.atom.match(dep_pkg) and
+ len(reasons) ==
1 and
+ not
reasons.get("slot conflict", True)):
+
self._dynamic_config._skip_restart = True
+ return 0
+
self._dynamic_config._backtrack_infos["missing dependency"] = dep
self._dynamic_config._need_restart = True
if debug:
diff --git a/lib/portage/tests/resolver/test_aggressive_backtrack_downgrade.py
b/lib/portage/tests/resolver/test_aggressive_backtrack_downgrade.py
new file mode 100644
index 0..fbe85dc89
--- /dev/null
+++ b/lib/portage/tests/resolver/test_aggressive_backtrack_downgrade.py
@@ -0,0 +1,91 @@
+# Copyright 2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (ResolverPlayground,
+ ResolverPlaygroundTestCase)
+
+class AgressiveBacktrackDowngradeTestCase(TestCase):
+
+ def testAgressiveBacktrackDowngrade(self):
+
+ ebuilds = {
+ 'www-client/firefox-69.0' : {
+ 'EAPI': '7',
+ 'RDEPEND':
'=media-libs/libvpx-1.7*:0=[postproc] media-video/ffmpeg'
+ },
+
+ 'www-client/firefox-60.9.0' : {
+ 'EAPI': '7',
+ 'RDEPEND': ''
+ },
+
+ 'media-libs/libvpx-1.8.0' : {
+ 'EAPI': '7',
+ 'SLOT' : '0/6',
+ 'IUSE': 'postproc',
+ },
+
+ 'media-libs/libvpx-1.7.0' : {
+ 'EAPI': '7',
+ 'SLOT' : '0/5',
+ 'IUSE': '+postproc',
+ },
+
+ 'media-libs/libvpx-1.6.0' : {
+ 'EAPI': '7',
+ 'SLOT' : '0/4',
+ 'IUSE': 'postproc',
+ },
+
+ 'media-video/ffmpeg-4.2' : {
+ 'EAPI': '7',
+ 'RDEPEND':
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/emaint/modules/sync/, lib/portage/dbapi/, ...
commit: 8084f01891f4110894054f999acc21b7a3899833
Author: Zac Medico gentoo org>
AuthorDate: Sat May 18 21:53:31 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Sat May 18 21:56:37 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8084f018
Use _safe_loop where applicable
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/PollScheduler.py | 6 +++---
lib/_emerge/depgraph.py | 4 ++--
lib/portage/dbapi/vartree.py | 19 +++
lib/portage/emaint/modules/sync/sync.py | 6 +++---
lib/portage/package/ebuild/_spawn_nofetch.py | 6 +++---
lib/portage/package/ebuild/doebuild.py | 19 +++
6 files changed, 25 insertions(+), 35 deletions(-)
diff --git a/lib/_emerge/PollScheduler.py b/lib/_emerge/PollScheduler.py
index 569879b36..e90830885 100644
--- a/lib/_emerge/PollScheduler.py
+++ b/lib/_emerge/PollScheduler.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2013 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
try:
@@ -7,6 +7,7 @@ except ImportError:
import dummy_threading as threading
import portage
+from portage.util.futures import asyncio
from portage.util._async.SchedulerInterface import SchedulerInterface
from portage.util._eventloop.EventLoop import EventLoop
from portage.util._eventloop.global_event_loop import global_event_loop
@@ -38,8 +39,7 @@ class PollScheduler(object):
elif main:
self._event_loop = global_event_loop()
else:
- self._event_loop = (portage._internal_caller and
- global_event_loop() or EventLoop(main=False))
+ self._event_loop = asyncio._safe_loop()
self._sched_iface = SchedulerInterface(self._event_loop,
is_background=self._is_background)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 32b2d7da3..3e99ac077 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -47,6 +47,7 @@ from portage.util import cmp_sort_key, writemsg,
writemsg_stdout
from portage.util import ensure_dirs
from portage.util import writemsg_level, write_atomic
from portage.util.digraph import digraph
+from portage.util.futures import asyncio
from portage.util._async.TaskScheduler import TaskScheduler
from portage.util._eventloop.EventLoop import EventLoop
from portage.util._eventloop.global_event_loop import global_event_loop
@@ -605,8 +606,7 @@ class depgraph(object):
self._select_atoms = self._select_atoms_highest_available
self._select_package = self._select_pkg_highest_available
- self._event_loop = (portage._internal_caller and
- global_event_loop() or EventLoop(main=False))
+ self._event_loop = asyncio._safe_loop()
self._select_atoms_parent = None
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 63389f9a3..e2fce7736 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2018 Gentoo Foundation
+# Copyright 1998-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import division, unicode_literals
@@ -58,6 +58,7 @@ from portage.exception import CommandNotFound, \
InvalidData, InvalidLocation, InvalidPackageName, \
FileNotFound, PermissionDenied, UnsupportedAPIException
from portage.localization import _
+from portage.util.futures import asyncio
from portage import abssymlink, _movefile, bsd_chflags
@@ -1406,8 +1407,7 @@ class vardbapi(dbapi):
# Do work via the global event loop, so that it can be
used
# for indication of progress during the search (bug
#461412).
- event_loop = (portage._internal_caller and
- global_event_loop() or EventLoop(main=False))
+ event_loop = asyncio._safe_loop()
root = self._vardb._eroot
def search_pkg(cpv, search_future):
@@ -1998,8 +1998,7 @@ class dblink(object):
if self._scheduler is None:
# We create a scheduler instance and use it to
# log unmerge output separately from merge output.
- self._scheduler =
SchedulerInterface(portage._internal_caller and
- global_event_loop() or EventLoop(main=False))
+ self._scheduler =
SchedulerInterface(asyncio._safe_loop())
if self.settings.get("PORTAGE_BACKGROUND") == "subprocess":
if self.settings.get("PORTAGE_BACKGROUND_UNMERGE") ==
"1":
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/
commit: bbfc36befdeed60f29c17d80d7766fd0da402d61
Author: Zac Medico gentoo org>
AuthorDate: Tue Jan 8 04:45:52 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Jan 8 09:12:11 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bbfc36be
pid-sandbox: whitelist selected pkg_* phases (bug 673794)
Whitelist the same phases that are whitelisted for FEATURES=cgroup,
since pid-sandbox is less valuable and is likely to have unintended
consenquences during these phases.
Bug: https://bugs.gentoo.org/673794
Reviewed-by: Michał Górny gentoo.org>
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/AbstractEbuildProcess.py | 9 ++---
lib/portage/package/ebuild/doebuild.py | 8 ++--
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/lib/_emerge/AbstractEbuildProcess.py
b/lib/_emerge/AbstractEbuildProcess.py
index bda0bd83f..ddf04e9b3 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -22,6 +22,10 @@ from portage.util.futures import asyncio
from portage.util._pty import _create_pty_or_pipe
from portage.util import apply_secpass_permissions
+portage.proxy.lazyimport.lazyimport(globals(),
+ 'portage.package.ebuild.doebuild:_global_pid_phases',
+)
+
class AbstractEbuildProcess(SpawnProcess):
__slots__ = ('phase', 'settings',) + \
@@ -30,7 +34,6 @@ class AbstractEbuildProcess(SpawnProcess):
_phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',)
_phases_interactive_whitelist = ('config',)
- _phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm',
'config')
# Number of milliseconds to allow natural exit of the ebuild
# process after it has called the exit command via IPC. It
@@ -71,7 +74,7 @@ class AbstractEbuildProcess(SpawnProcess):
# Check if the cgroup hierarchy is in place. If it's not, mount
it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
- and self.phase not in
self._phases_without_cgroup):
+ and self.phase not in _global_pid_phases):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index baebb9a27..f11923595 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2018 Gentoo Authors
+# Copyright 2010-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import unicode_literals
@@ -110,6 +110,9 @@ _ipc_phases = frozenset([
"preinst", "postinst", "prerm", "postrm",
])
+# phases which execute in the global PID namespace
+_global_pid_phases = frozenset(['preinst', 'postinst', 'prerm', 'postrm',
'config'])
+
# phases in which networking access is allowed
_networked_phases = frozenset([
# for VCS fetching
@@ -153,7 +156,8 @@ def _doebuild_spawn(phase, settings, actionmap=None,
**kwargs):
kwargs['networked'] = 'network-sandbox' not in settings.features or \
phase in _networked_phases or \
'network-sandbox' in settings['PORTAGE_RESTRICT'].split()
- kwargs['pidns'] = 'pid-sandbox' in settings.features
+ kwargs['pidns'] = ('pid-sandbox' in settings.features and
+ phase not in _global_pid_phases)
if phase == 'depend':
kwargs['droppriv'] = 'userpriv' in settings.features
