[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: b16ab6c7088cfde7664f1728542545e6dafce99d Author: Bryan Gardiner khumba net> AuthorDate: Sat May 11 03:18:56 2024 + Commit: Sam James gentoo org> CommitDate: Mon May 13 05:43:48 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b16ab6c7 depgraph.py: Fix the URL for the changed-deps wiki page Signed-off-by: Bryan Gardiner khumba.net> Signed-off-by: Sam James gentoo.org> lib/_emerge/depgraph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 6853ec4911..9673d85f87 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -1181,7 +1181,7 @@ class depgraph: "NOTE: Refer to the following page for more information about dependency", " change(s) without revision bump:", "", -" https://wiki.gentoo.org/wiki/Project:Portage/Changed_Deps;, +" https://wiki.gentoo.org/wiki/Project:Portage/Changed_dependencies;, "", " In order to suppress reports about dependency changes, add", " --changed-deps-report=n to the EMERGE_DEFAULT_OPTS variable in",
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 7c9bb2caeaff59232cb44e84b585db3feb2dd5fe Author: Sam James gentoo org> AuthorDate: Sat May 4 12:05:17 2024 + Commit: Sam James gentoo org> CommitDate: Sat May 4 12:05:17 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c9bb2ca actions: handle pkgmoves for autoconf, automake, libtool Signed-off-by: Sam James gentoo.org> lib/_emerge/actions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index d36a799244..b93c08aa07 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -2070,11 +2070,11 @@ def action_info(settings, trees, myopts, myfiles): append(ccache_str) myvars = [ -"sys-devel/autoconf", -"sys-devel/automake", +"dev-build/autoconf", +"dev-build/automake", "virtual/os-headers", "sys-devel/binutils", -"sys-devel/libtool", +"dev-build/libtool", "dev-lang/python", ] myvars += portage.util.grabfile(settings["PORTDIR"] + "/profiles/info_pkgs")
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 08a2bc3800ea2e997716903244ec041339b45b06
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 24 22:11:54 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Mar 28 14:47:49 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=08a2bc38
BinpkgPrefetcher: Emit eerror message for binarytree inject failure
Since it is confusing when emerge exits due to an error that only shows
in the fetch log, emit an eerror message when binarytree inject fails:
>>> Running pre-merge checks for sys-libs/glibc-2.38-r10
* Fetching in the background:
* /var/cache/binpkgs/sys-libs/glibc/glibc-2.38-r10-5.gpkg.tar.partial
* To view fetch progress, run in another terminal:
* tail -f /var/log/emerge-fetch.log
>>> Failed to emerge sys-libs/glibc-2.38-r10
* Messages for package sys-libs/glibc-2.38-r10:
* Binary package is not usable:
* !!!
* gpg: keyblock resource '/etc/portage/gnupg/pubring.kbx': No such file
or directory
* [GNUPG:] ERROR add_keyblock_resource 33587281
* [GNUPG:] PLAINTEXT 74 0
* [GNUPG:] NEWSIG
* gpg: Signature made Wed 20 Mar 2024 10:34:45 PM CET
* gpg:using RSA key
534E4209AB49EEE1C19D96162C44695DB9F6043D
* [GNUPG:] ERROR keydb_search 33554445
* [GNUPG:] ERROR keydb_search 33554445
* [GNUPG:] ERRSIG 2C44695DB9F6043D 1 10 01 1710970485 9
534E4209AB49EEE1C19D96162C44695DB9F6043D
* [GNUPG:] NO_PUBKEY 2C44695DB9F6043D
* !!! Invalid binary package:
'/var/cache/binpkgs/sys-devel/binutils/binutils-2.41-r5-4.gpkg.tar.partial',
GPG verify failed
Fixes: 0ff49114cec7 ("binarytree: Handle inject failures")
Bug: https://bugs.gentoo.org/927632
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/BinpkgPrefetcher.py | 35 ++-
1 file changed, 30 insertions(+), 5 deletions(-)
diff --git a/lib/_emerge/BinpkgPrefetcher.py b/lib/_emerge/BinpkgPrefetcher.py
index ed68d2852c..a8af30ca80 100644
--- a/lib/_emerge/BinpkgPrefetcher.py
+++ b/lib/_emerge/BinpkgPrefetcher.py
@@ -7,7 +7,10 @@ import sys
from _emerge.BinpkgFetcher import BinpkgFetcher
from _emerge.CompositeTask import CompositeTask
from _emerge.BinpkgVerifier import BinpkgVerifier
+import portage
from portage import os
+from portage.elog import messages as elog_messages
+from portage.util import no_color
class BinpkgPrefetcher(CompositeTask):
@@ -48,6 +51,7 @@ class BinpkgPrefetcher(CompositeTask):
self.wait()
return
+injected_pkg = None
stdout_orig = sys.stdout
stderr_orig = sys.stderr
out = io.StringIO()
@@ -67,12 +71,33 @@ class BinpkgPrefetcher(CompositeTask):
output_value = out.getvalue()
if output_value:
-self.scheduler.output(
-output_value,
-log_path=self.scheduler.fetch.log_file,
-background=self.background,
-)
+if injected_pkg is None:
+msg = ["Binary package is not usable:"]
+msg.extend("\t" + line for line in
output_value.splitlines())
+self._elog("eerror", msg)
+else:
+self.scheduler.output(
+output_value,
+log_path=self.scheduler.fetch.log_file,
+background=self.background,
+)
self._current_task = None
self.returncode = 1 if injected_pkg is None else os.EX_OK
self.wait()
+
+def _elog(self, elog_funcname, lines, phase="other"):
+out = io.StringIO()
+elog_func = getattr(elog_messages, elog_funcname)
+global_havecolor = portage.output.havecolor
+try:
+portage.output.havecolor = not no_color(self._bintree.settings)
+for line in lines:
+elog_func(line, phase=phase, key=self.pkg.cpv, out=out)
+finally:
+portage.output.havecolor = global_havecolor
+msg = out.getvalue()
+if msg:
+self.scheduler.output(
+msg, background=self.background,
log_path=self.scheduler.fetch.log_file
+)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: c3ed026be40b4cf857b33854e70f59f7e016e3b5
Author: Zac Medico gentoo org>
AuthorDate: Sat Mar 9 23:23:28 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Mar 9 23:24:44 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c3ed026b
_setup_repo_revisions: Skip async_aux_get for INHERITED
Since INHERITED is always available for the ebuild environment,
get it from settings.configdict["pkg"]["INHERITED"].
Fixes: d55d415a4f68 ("EbuildPhase/EbuildBinpkg: Ensure PKGDIR subdirectory
permissions")
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildPhase.py | 13 +++--
1 file changed, 3 insertions(+), 10 deletions(-)
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 73c284fe41..9e5268ff73 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -92,17 +92,10 @@ async def _setup_repo_revisions(settings):
and repo_name
and "PORTAGE_REPO_REVISIONS" not in settings.configdict["pkg"]
):
-inherits = frozenset(
-(
-await db.async_aux_get(
-settings.mycpv,
-["INHERITED"],
-myrepo=repo_name,
-)
-)[0].split()
-)
repo = db.repositories[repo_name]
-ec_dict = repo.eclass_db.get_eclass_data(inherits)
+ec_dict = repo.eclass_db.get_eclass_data(
+settings.configdict["pkg"]["INHERITED"].split()
+)
referenced_repos = {repo.name: repo}
for ec_info in ec_dict.values():
ec_repo = db.repositories.get_repo_for_location(
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
commit: 6f9a10d38259dd61b948837e193b047464791845
Author: Zac Medico gentoo org>
AuthorDate: Mon Mar 4 05:31:08 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Mar 4 05:33:44 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6f9a10d3
SpawnProcess: Optimize away null input for create_pipe=False
When create_pipe=False support was added in commit e8b31c86eaed,
a null input file descriptor was used for PipeLogger and BuildLogger
instances. Optimize this away, eliminating the unnecessary loggers.
Fixes: e8b31c86eaed ("ForkProcess: Prevent redundant pipe and set_term_size
recursion")
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/SpawnProcess.py| 51 +-
lib/portage/util/_async/ForkProcess.py | 7 ++---
2 files changed, 28 insertions(+), 30 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 9fc12c42e5..513a7b2fe4 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -79,10 +79,7 @@ class SpawnProcess(SubProcess):
# SpawnProcess will have created a pipe earlier, so it
# would be redundant to do it here (it could also trigger
# spawn recursion via set_term_size as in bug 923750).
-# Use /dev/null for master_fd, triggering early return
-# of _main, followed by _async_waitpid.
-# TODO: Optimize away the need for master_fd here.
-master_fd = os.open(os.devnull, os.O_RDONLY)
+master_fd = None
slave_fd = None
can_log = False
@@ -166,23 +163,27 @@ class SpawnProcess(SubProcess):
self._registered = True
def _start_main_task(self, pr, log_file_path=None, stdout_fd=None):
-build_logger = BuildLogger(
-env=self.env,
-log_path=log_file_path,
-log_filter_file=self.log_filter_file,
-scheduler=self.scheduler,
-)
-build_logger.start()
-
-pipe_logger = PipeLogger(
-background=self.background,
-scheduler=self.scheduler,
-input_fd=pr,
-log_file_path=build_logger.stdin,
-stdout_fd=stdout_fd,
-)
-
-pipe_logger.start()
+if pr is None:
+build_logger = None
+pipe_logger = None
+else:
+build_logger = BuildLogger(
+env=self.env,
+log_path=log_file_path,
+log_filter_file=self.log_filter_file,
+scheduler=self.scheduler,
+)
+build_logger.start()
+
+pipe_logger = PipeLogger(
+background=self.background,
+scheduler=self.scheduler,
+input_fd=pr,
+log_file_path=build_logger.stdin,
+stdout_fd=stdout_fd,
+)
+
+pipe_logger.start()
self._main_task_cancel = functools.partial(
self._main_cancel, build_logger, pipe_logger
@@ -198,18 +199,18 @@ class SpawnProcess(SubProcess):
await self._pty_ready
self._pty_ready = None
try:
-if pipe_logger.poll() is None:
+if pipe_logger is not None and pipe_logger.poll() is None:
await pipe_logger.async_wait()
-if build_logger.poll() is None:
+if build_logger is not None and build_logger.poll() is None:
await build_logger.async_wait()
except asyncio.CancelledError:
self._main_cancel(build_logger, pipe_logger)
raise
def _main_cancel(self, build_logger, pipe_logger):
-if pipe_logger.poll() is None:
+if pipe_logger is not None and pipe_logger.poll() is None:
pipe_logger.cancel()
-if build_logger.poll() is None:
+if build_logger is not None and build_logger.poll() is None:
build_logger.cancel()
def _main_exit(self, main_task):
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index ebcbd94107..e6cfdefb88 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -91,11 +91,8 @@ class ForkProcess(SpawnProcess):
# When called via process.spawn, SpawnProcess
# will have created a pipe earlier, so it would be
# redundant to do it here (it could also trigger spawn
-# recursion via set_term_size as in bug 923750). Use
-# /dev/null for master_fd, triggering early return
-# of _main, followed by _async_waitpid.
-# TODO: Optimize away the need for master_fd here.
-master_fd = os.open(os.devnull, os.O_RDONLY)
+# recursion via
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/dbapi/, lib/portage/util/_async/, ...
commit: acb69a6f234bd412e95e76f5c1db1b1f5b8e1dc5
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 24 03:52:47 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 24 04:23:27 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=acb69a6f
SchedulerInterface/PollScheduler: Add _loop property
This allows async_aux_get to easily verify the identity of
the underlying loop so that this assertion will not fail:
_start_with_metadata
(settings.configdict["pkg"]["SRC_URI"],) = aux_get_task.future.result()
File "/usr/lib/python3.11/site-packages/portage/dbapi/porttree.py", line 786,
in async_aux_get
raise AssertionError(
AssertionError: async_aux_get called from thread <_MainThread(MainThread,
started 281473559502880)> with loop <_emerge.Scheduler.Scheduler._iface_class
object at 0x8e3a8840>
Terminated
Fixes: 389bb304abf5 ("async_aux_get: Use EbuildMetadataPhase deallocate_config
future")
Bug: https://bugs.gentoo.org/925333
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/PollScheduler.py| 9 -
lib/portage/dbapi/porttree.py | 2 +-
lib/portage/tests/ebuild/test_doebuild_spawn.py | 3 ++-
lib/portage/tests/ebuild/test_ipc_daemon.py | 3 ++-
lib/portage/util/_async/SchedulerInterface.py | 9 -
5 files changed, 21 insertions(+), 5 deletions(-)
diff --git a/lib/_emerge/PollScheduler.py b/lib/_emerge/PollScheduler.py
index e5dffd8afa..b5bfd20b76 100644
--- a/lib/_emerge/PollScheduler.py
+++ b/lib/_emerge/PollScheduler.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import threading
@@ -39,6 +39,13 @@ class PollScheduler:
self._event_loop, is_background=self._is_background
)
+@property
+def _loop(self):
+"""
+Returns the real underlying asyncio loop.
+"""
+return self._event_loop._loop
+
def _is_background(self):
return self._background
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 4eebe1183f..de6aa5c82b 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -775,7 +775,7 @@ class portdbapi(dbapi):
try:
if (
threading.current_thread() is threading.main_thread()
-and loop is asyncio._safe_loop()
+and loop._loop is asyncio._safe_loop()._loop
):
# In this case use self._doebuild_settings_lock to manage
concurrency.
deallocate_config = loop.create_future()
diff --git a/lib/portage/tests/ebuild/test_doebuild_spawn.py
b/lib/portage/tests/ebuild/test_doebuild_spawn.py
index 9fb2c7fdd4..cac844f8fb 100644
--- a/lib/portage/tests/ebuild/test_doebuild_spawn.py
+++ b/lib/portage/tests/ebuild/test_doebuild_spawn.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2015 Gentoo Foundation
+# Copyright 2010-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import textwrap
@@ -86,6 +86,7 @@ class DoebuildSpawnTestCase(TestCase):
open(os.path.join(settings["T"], "environment"), "wb").close()
scheduler = SchedulerInterface(global_event_loop())
+self.assertTrue(scheduler._loop is global_event_loop()._loop)
for phase in ("_internal_test",):
# Test EbuildSpawnProcess by calling doebuild.spawn() with
# returnpid=False. This case is no longer used by portage
diff --git a/lib/portage/tests/ebuild/test_ipc_daemon.py
b/lib/portage/tests/ebuild/test_ipc_daemon.py
index 0beb69ddfc..b8777fe94b 100644
--- a/lib/portage/tests/ebuild/test_ipc_daemon.py
+++ b/lib/portage/tests/ebuild/test_ipc_daemon.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2023 Gentoo Authors
+# Copyright 2010-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import tempfile
@@ -77,6 +77,7 @@ class IpcDaemonTestCase(TestCase):
task_scheduler = TaskScheduler(
iter([daemon, proc]), max_jobs=2, event_loop=event_loop
)
+self.assertTrue(task_scheduler._loop is event_loop._loop)
self.received_command = False
diff --git a/lib/portage/util/_async/SchedulerInterface.py
b/lib/portage/util/_async/SchedulerInterface.py
index 43a42adff2..485958491a 100644
--- a/lib/portage/util/_async/SchedulerInterface.py
+++ b/lib/portage/util/_async/SchedulerInterface.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2021 Gentoo Authors
+# Copyright 2012-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import gzip
@@ -49,6 +49,13 @@ class SchedulerInterface(SlotObject):
for k in self._event_loop_attrs:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: d06515e1424be60c8ed7af6ed2a80d56145b85c5
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 24 13:28:22 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 24 13:39:29 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d06515e1
_dynamic_deps_preload: Fix settings reference
Use the settings reference from the config pool as
intended to prevent a KeyError similar to bug 924319
but triggered by emerge --dynamic-deps.
Fixes: f9ea958018c0 ("MetadataRegen: Use EbuildMetadataPhase deallocate_config")
Bug: https://bugs.gentoo.org/925350
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index ea96bd58c4..6853ec4911 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -783,7 +783,7 @@ class depgraph:
ebuild_hash=ebuild_hash,
portdb=portdb,
repo_path=repo_path,
-settings=portdb.doebuild_settings,
+settings=settings,
deallocate_config=deallocate_config,
)
proc.addExitListener(self._dynamic_deps_proc_exit(pkg,
fake_vartree))
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 789493d521eb09f4aca2a8552b5262e2c0f5bc51 Author: Zac Medico gentoo org> AuthorDate: Tue Feb 13 08:19:59 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 21 15:27:31 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=789493d5 EbuildMetadataPhase: Migrate to _async_start Bug: https://bugs.gentoo.org/923841 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/EbuildMetadataPhase.py | 17 ++--- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/_emerge/EbuildMetadataPhase.py b/lib/_emerge/EbuildMetadataPhase.py index 249086f8af..fc64d84c94 100644 --- a/lib/_emerge/EbuildMetadataPhase.py +++ b/lib/_emerge/EbuildMetadataPhase.py @@ -46,6 +46,12 @@ class EbuildMetadataPhase(SubProcess): _files_dict = slot_dict_class(_file_names, prefix="") def _start(self): +asyncio.ensure_future( +self._async_start(), loop=self.scheduler +).add_done_callback(self._async_start_done) +self._registered = True + +async def _async_start(self): ebuild_path = self.ebuild_hash.location with open( @@ -116,7 +122,6 @@ class EbuildMetadataPhase(SubProcess): self._raw_metadata = [] files.ebuild = master_fd self.scheduler.add_reader(files.ebuild, self._output_handler) -self._registered = True retval = portage.doebuild( ebuild_path, @@ -150,16 +155,6 @@ class EbuildMetadataPhase(SubProcess): self._proc = retval -asyncio.ensure_future( -self._async_start(), loop=self.scheduler -).add_done_callback(self._async_start_done) - -async def _async_start(self): -# Call async check_locale here for bug 923841, but code -# also needs to migrate from _start to here, including -# the self.deallocate_config set_result call. -pass - def _async_start_done(self, future): future.cancelled() or future.result() if not self._was_cancelled() and future.cancelled():
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 69fbd8f9f76df32d9bc72510e5d9348eb8f059bc
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 04:04:53 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:30 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=69fbd8f9
EbuildMetadataPhase: Add deallocate_config future
Use a deallocate_config future to release self.settings when
it is no longer needed. It's necessary to manage concurrency
since commit c95fc64abf96 because mutation of self.settings
is no longer limited to the EbuildMetadataPhase _start method,
where exclusive access was guaranteed within the main thread.
Add support to the isAlive() method to detect when the
EbuildMetadataPhase has started but the pid is not yet
available (due to async_check_locale usage from commit
c95fc64abf96). This can be used to check if an
EbuildMetadataPhase instance has been successfully started
so that it can be relied upon to set the result of the
deallocate_config future.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py | 34 ++
lib/_emerge/SubProcess.py | 5 -
2 files changed, 38 insertions(+), 1 deletion(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index f4f685e81c..249086f8af 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -14,6 +14,7 @@ from portage import os
from portage import _encodings
from portage import _unicode_decode
from portage import _unicode_encode
+from portage.util.futures import asyncio
import fcntl
@@ -33,6 +34,7 @@ class EbuildMetadataPhase(SubProcess):
"portdb",
"repo_path",
"settings",
+"deallocate_config",
"write_auxdb",
) + (
"_eapi",
@@ -127,6 +129,15 @@ class EbuildMetadataPhase(SubProcess):
returnproc=True,
)
settings.pop("PORTAGE_PIPE_FD", None)
+# At this point we can return settings to the caller
+# since we never use it for anything more than an
+# eapi_invalid call after this, and eapi_invalid is
+# insensitive to concurrent modifications.
+if (
+self.deallocate_config is not None
+and not self.deallocate_config.cancelled()
+):
+self.deallocate_config.set_result(settings)
os.close(slave_fd)
null_input.close()
@@ -139,6 +150,29 @@ class EbuildMetadataPhase(SubProcess):
self._proc = retval
+asyncio.ensure_future(
+self._async_start(), loop=self.scheduler
+).add_done_callback(self._async_start_done)
+
+async def _async_start(self):
+# Call async check_locale here for bug 923841, but code
+# also needs to migrate from _start to here, including
+# the self.deallocate_config set_result call.
+pass
+
+def _async_start_done(self, future):
+future.cancelled() or future.result()
+if not self._was_cancelled() and future.cancelled():
+self.cancel()
+self._was_cancelled()
+
+if self.deallocate_config is not None and not
self.deallocate_config.done():
+self.deallocate_config.set_result(self.settings)
+
+if self.returncode is not None:
+self._unregister()
+self.wait()
+
def _output_handler(self):
while True:
buf = self._read_buf(self._files.ebuild)
diff --git a/lib/_emerge/SubProcess.py b/lib/_emerge/SubProcess.py
index 029bbc3f44..057e0adc24 100644
--- a/lib/_emerge/SubProcess.py
+++ b/lib/_emerge/SubProcess.py
@@ -18,9 +18,12 @@ class SubProcess(AbstractPollTask):
# we've sent a kill signal to our subprocess.
_cancel_timeout = 1 # seconds
+def isAlive(self):
+return (self._registered or self.pid is not None) and self.returncode
is None
+
@property
def pid(self):
-return self._proc.pid
+return None if self._proc is None else self._proc.pid
def _poll(self):
# Simply rely on _async_waitpid_cb to set the returncode.
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: f9ea958018c02b3ce07761c2d965260498add2af
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 04:04:34 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:30 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f9ea9580
MetadataRegen: Use EbuildMetadataPhase deallocate_config
For EbuildMetadataPhase consumers like MetadataRegen and
depgraph, store a pool of config instances in a config_pool
list, and return instaces to the list when the deallocate_config
future is done.
Bug: https://bugs.gentoo.org/924319
Fixes: c95fc64abf96 ("EbuildPhase: async_check_locale")
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/MetadataRegen.py | 16 ++--
lib/_emerge/depgraph.py | 11 +++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/lib/_emerge/MetadataRegen.py b/lib/_emerge/MetadataRegen.py
index d29722b94c..538a94b450 100644
--- a/lib/_emerge/MetadataRegen.py
+++ b/lib/_emerge/MetadataRegen.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
@@ -44,6 +44,7 @@ class MetadataRegen(AsyncScheduler):
valid_pkgs = self._valid_pkgs
cp_set = self._cp_set
consumer = self._consumer
+config_pool = []
portage.writemsg_stdout("Regenerating cache entries...\n")
for cp in self._cp_iter:
@@ -73,12 +74,23 @@ class MetadataRegen(AsyncScheduler):
consumer(cpv, repo_path, metadata, ebuild_hash,
True)
continue
+if config_pool:
+settings = config_pool.pop()
+else:
+settings = portage.config(clone=portdb.settings)
+
+deallocate_config = self.scheduler.create_future()
+deallocate_config.add_done_callback(
+lambda future: config_pool.append(future.result())
+)
+
yield EbuildMetadataPhase(
cpv=cpv,
ebuild_hash=ebuild_hash,
portdb=portdb,
repo_path=repo_path,
-settings=portdb.doebuild_settings,
+settings=settings,
+deallocate_config=deallocate_config,
write_auxdb=self._write_auxdb,
)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 1674fa289e..70b83ee1f4 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -754,6 +754,7 @@ class depgraph:
def _dynamic_deps_preload(self, fake_vartree):
portdb = fake_vartree._portdb
+config_pool = []
for pkg in fake_vartree.dbapi:
self._spinner_update()
self._dynamic_config._package_tracker.add_installed_pkg(pkg)
@@ -768,12 +769,22 @@ class depgraph:
if metadata is not None:
fake_vartree.dynamic_deps_preload(pkg, metadata)
else:
+if config_pool:
+settings = config_pool.pop()
+else:
+settings = portage.config(clone=portdb.settings)
+
+deallocate_config = portdb._event_loop.create_future()
+deallocate_config.add_done_callback(
+lambda future: config_pool.append(future.result())
+)
proc = EbuildMetadataPhase(
cpv=pkg.cpv,
ebuild_hash=ebuild_hash,
portdb=portdb,
repo_path=repo_path,
settings=portdb.doebuild_settings,
+deallocate_config=deallocate_config,
)
proc.addExitListener(self._dynamic_deps_proc_exit(pkg,
fake_vartree))
yield proc
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/, bin/, man/
commit: 749d231d57fb64d3f2e909dfbba26ca7c1ab9d0e
Author: YiFei Zhu gmail com>
AuthorDate: Mon Jan 29 08:50:28 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 21 02:13:04 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=749d231d
estrip: Support debug info deduplication with sys-devel/dwz
This this gates behind FEATURES=dedupdebug, and packages can
opt-out with RESTRICT=dedupdebug, though I'm not sure why a package
would need to opt this out, unless dwz could break something I'm not
aware of...
Bug: https://bugs.gentoo.org/906368
Signed-off-by: YiFei Zhu gmail.com>
Closes: https://github.com/gentoo/portage/pull/1245
Signed-off-by: Sam James gentoo.org>
bin/estrip | 35 +--
lib/_emerge/EbuildPhase.py | 1 +
lib/portage/const.py | 1 +
man/ebuild.5 | 3 +++
man/make.conf.5| 8 +++-
5 files changed, 45 insertions(+), 3 deletions(-)
diff --git a/bin/estrip b/bin/estrip
index 3ac6a16927..2d9d50922a 100755
--- a/bin/estrip
+++ b/bin/estrip
@@ -15,8 +15,8 @@ exp_tf() {
eval ${var}_${flag}=$(tf has ${flag} ${!var})
done
}
-exp_tf FEATURES compressdebug installsources nostrip splitdebug xattr
-exp_tf PORTAGE_RESTRICT binchecks installsources splitdebug strip
+exp_tf FEATURES compressdebug dedupdebug installsources nostrip splitdebug
xattr
+exp_tf PORTAGE_RESTRICT binchecks dedupdebug installsources splitdebug strip
if ! ___eapi_has_prefix_variables; then
EPREFIX= ED=${D}
@@ -201,6 +201,10 @@ fi
[[ ${debugedit} ]] && debugedit_found=true || debugedit_found=false
debugedit_warned=false
+dwz=$(type -P dwz)
+[[ ${dwz} ]] && dwz_found=true || dwz_found=false
+dwz_warned=false
+
__multijob_init
# Setup ${T} filesystem layout that we care about.
@@ -248,6 +252,32 @@ __try_symlink() {
die "failed to create symlink '${name}'"
}
+# Usage: dedup_elf_debug
+dedup_elf_debug() {
+ ${FEATURES_dedupdebug} || return 0
+ ${PORTAGE_RESTRICT_dedupdebug} && return 0
+
+ debug-print-function "${FUNCNAME}" "$@"
+
+ if ! ${dwz_found} ; then
+ if ! ${dwz_warned} ; then
+ dwz_warned=true
+ ewarn "FEATURES=dedupdebug is enabled but the dwz
binary could not be"
+ ewarn "found. This feature will not work unless dwz is
installed!"
+ fi
+ return 0
+ fi
+
+ local src=$1 # File to dedup debug symbols
+ local inode_dedupdebug=$2 # Temp path for hard link tracking
+
+ # We already dedupdebug-ed this inode.
+ [[ -L ${inode_dedupdebug} ]] && return 0
+
+ "${dwz}" -- "${src}"
+ touch "${inode_dedupdebug}"
+}
+
# Usage: save_elf_debug [splitdebug]
save_elf_debug() {
${FEATURES_splitdebug} || return 0
@@ -355,6 +385,7 @@ process_elf() {
xt_data=$(dump_xattrs "${x}")
fi
save_elf_sources "${x}"
+ dedup_elf_debug "${x}" "${inode_link}_dedupdebug"
fi
if ${strip_this} ; then
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index c81bf54a81..3b366f39c7 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -67,6 +67,7 @@ class EbuildPhase(CompositeTask):
_features_display = (
"ccache",
"compressdebug",
+"dedupdebug",
"distcc",
"fakeroot",
"installsources",
diff --git a/lib/portage/const.py b/lib/portage/const.py
index 1f468a1abf..2154213b7b 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -176,6 +176,7 @@ SUPPORTED_FEATURES = frozenset(
"compressdebug",
"compress-index",
"config-protect-if-modified",
+"dedupdebug",
"digest",
"distcc",
"distlocks",
diff --git a/man/ebuild.5 b/man/ebuild.5
index aad189bb00..f849f20a29 100644
--- a/man/ebuild.5
+++ b/man/ebuild.5
@@ -754,6 +754,9 @@ see the \fBQA CONTROL VARIABLES\fR section for more
specific exemptions.
.I bindist
Distribution of built packages is restricted.
.TP
+.I dedupdebug
+Disables dedupdebug for specific packages.
+.TP
.I fetch
like \fImirror\fR but the files will not be fetched via \fBSRC_URI\fR either.
.TP
diff --git a/man/make.conf.5 b/man/make.conf.5
index 20dc9cb52d..23d8408544 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -465,13 +465,19 @@ redundant on\-the\-fly compression. The resulting file
will be called
.B compressdebug
Compress the debug sections in the split debug files with zlib to save
space. See \fBsplitdebug\fR for general split debug
-information (upon which this feature depends).
+information (upon which this feature depends). See also \fBdedupdebug\fR
+for further debug info size reduction.
.TP
.B config\-protect\-if\-modified
This causes the \fBCONFIG_PROTECT\fR behavior to be skipped
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/
commit: 67663e586320347e604727aef006bfe027bcd470
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 12 02:10:52 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 12 13:57:42 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=67663e58
EbuildBuild: Execute EbuildFetchonly in subprocess
Execute EbuildFetchonly in a subprocess since it needs to
run the event loop itself (even for pretend mode since it
may need to fetch mirror layouts as reported in bug 702154).
Also remove obsolete loop.is_running() case in doebuild
related to bug 601252, since it will fail if a mirror
layout file needs to be fetched, so we should just assume
that the event loop is not running.
Set fetched = 2 for listonly (pretend) mode in the fetch
function so that EbuildFetchonly will properly report
success for this mode.
Pass ebuild_path to EbuildFetchonly, and also remove
setcpv call from EbuildFetchonly since it's called earlier
in EbuildBuild.
Bug: https://bugs.gentoo.org/924287
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildBuild.py | 39 +-
lib/_emerge/EbuildFetchonly.py | 10 +++--
lib/portage/package/ebuild/doebuild.py | 30 --
lib/portage/package/ebuild/fetch.py| 1 +
4 files changed, 33 insertions(+), 47 deletions(-)
diff --git a/lib/_emerge/EbuildBuild.py b/lib/_emerge/EbuildBuild.py
index 81cbfdc085..d4a4c6dacb 100644
--- a/lib/_emerge/EbuildBuild.py
+++ b/lib/_emerge/EbuildBuild.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -22,6 +22,7 @@ from portage.package.ebuild.digestcheck import digestcheck
from portage.package.ebuild.doebuild import _check_temp_dir
from portage.package.ebuild._spawn_nofetch import SpawnNofetchWithoutBuilddir
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util.futures.executor.fork import ForkExecutor
from portage.util.path import first_existing
@@ -152,29 +153,25 @@ class EbuildBuild(CompositeTask):
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(
+ebuild_path=self._ebuild_path,
fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings,
)
-retval = fetcher.execute()
-if retval == os.EX_OK:
-self._current_task = None
-self.returncode = os.EX_OK
-self._async_wait()
-else:
-# For pretend mode, the convention it to execute
-# pkg_nofetch and return a successful exitcode.
-self._start_task(
-SpawnNofetchWithoutBuilddir(
-background=self.background,
-
portdb=self.pkg.root_config.trees[self._tree].dbapi,
-ebuild_path=self._ebuild_path,
-scheduler=self.scheduler,
-settings=self.settings,
+# Execute EbuildFetchonly in a subprocess since it needs to
+# run the event loop itself (even for pretend mode since it
+# may need to fetch mirror layouts as reported in bug 702154).
+self._start_task(
+AsyncTaskFuture(
+background=self.background,
+scheduler=self.scheduler,
+future=self.scheduler.run_in_executor(
+ForkExecutor(loop=self.scheduler), fetcher.execute
),
-self._default_final_exit,
-)
+),
+self._fetchonly_exit,
+)
return
quiet_setting = settings.get("PORTAGE_QUIET", False)
@@ -241,8 +238,12 @@ class EbuildBuild(CompositeTask):
self._start_task(pre_clean_phase, self._pre_clean_exit)
def _fetchonly_exit(self, fetcher):
+if not fetcher.cancelled and isinstance(fetcher, AsyncTaskFuture):
+# Set returncode from EbuildFetchonly.execute() result, since
+# it can fail if it can't resolve a mirror for a file.
+fetcher.returncode = fetcher.future.result()
self._final_exit(fetcher)
-if self.returncode != os.EX_OK:
+if not self.cancelled and self.returncode != os.EX_OK:
self.returncode = None
portdb = self.pkg.root_config.trees[self._tree].dbapi
self._start_task(
diff --git a/lib/_emerge/EbuildFetchonly.py b/lib/_emerge/EbuildFetchonly.py
index e887dd858d..c806122bea 100644
--- a/lib/_emerge/EbuildFetchonly.py
+++
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/
commit: a69c1b853a47346192950c91b088163490287350 Author: Zac Medico gentoo org> AuthorDate: Sat Feb 3 21:27:45 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 7 00:49:26 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a69c1b85 process.spawn: Use multiprocessing.Process for returnproc Use multiprocessing.Process for returnproc, so that fork will stop being used when python makes "spawn" the default multiprocessing start method. Continue to use _start_fork when returnproc is not enabled, for backward compatibility. Ultimately, it can be removed at the same time as the returnpid parameter. The _setup_pipes_after_fork wrapper prevents a "Bad file descriptor" error by making fd_pipes inheritable on exec for bug 923755. ForkProcess does not handle this because its target function does not necessarily exec. Bug: https://bugs.gentoo.org/916566 Bug: https://bugs.gentoo.org/923755 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/SpawnProcess.py | 4 +- lib/portage/process.py | 92 ++--- 2 files changed, 82 insertions(+), 14 deletions(-) diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py index 716e94d665..b63afae01c 100644 --- a/lib/_emerge/SpawnProcess.py +++ b/lib/_emerge/SpawnProcess.py @@ -241,7 +241,9 @@ class SpawnProcess(SubProcess): got_pty, master_fd, slave_fd = _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) -def _spawn(self, args: list[str], **kwargs) -> portage.process.Process: +def _spawn( +self, args: list[str], **kwargs +) -> portage.process.MultiprocessingProcess: spawn_func = portage.process.spawn if self._selinux_type is not None: diff --git a/lib/portage/process.py b/lib/portage/process.py index b223ecb887..20327b38bc 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -19,7 +19,7 @@ import warnings from dataclasses import dataclass from functools import lru_cache -from typing import Any, Optional, Callable +from typing import Any, Optional, Callable, Union from portage import os from portage import _encodings @@ -28,6 +28,7 @@ import portage portage.proxy.lazyimport.lazyimport( globals(), +"portage.util._async.ForkProcess:ForkProcess", "portage.util._eventloop.global_event_loop:global_event_loop", "portage.util.futures:asyncio", "portage.util:dump_traceback,writemsg,writemsg_level", @@ -296,12 +297,19 @@ class AbstractProcess: class Process(AbstractProcess): """ -An object that wraps OS processes created by spawn. -In the future, spawn will return objects of a different type -but with a compatible interface to this class, in order -to encapsulate implementation-dependent objects like -multiprocessing.Process which are designed to manage -the process lifecycle and need to persist until it exits. +An object that wraps OS processes which do not have an +associated multiprocessing.Process instance. Ultimately, +we need to stop using os.fork() to create these processes +because it is unsafe for threaded processes as discussed +in https://github.com/python/cpython/issues/84559. + +Note that if subprocess.Popen is used without pass_fds +or preexec_fn parameters, then it avoids using os.fork() +by instead using posix_spawn. This approach is not used +by spawn because it needs to execute python code prior +to exec, so it instead uses multiprocessing.Process, +which only uses os.fork() when the multiprocessing start +method is fork. """ def __init__(self, pid: int): @@ -461,7 +469,7 @@ def spawn( unshare_mount=False, unshare_pid=False, warn_on_large_env=False, -): +) -> Union[int, MultiprocessingProcess, list[int]]: """ Spawns a given command. @@ -479,8 +487,8 @@ def spawn( @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean -@param returnproc: Return a Process object for a successful spawn (conflicts with logfile parameter). -NOTE: This requires the caller to asynchronously wait for the Process. +@param returnproc: Return a MultiprocessingProcess instance (conflicts with logfile parameter). +NOTE: This requires the caller to asynchronously wait for the MultiprocessingProcess instance. @type returnproc: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @@ -626,7 +634,9 @@ def spawn( # fork, so that the result is cached in the main process. bool(groups) -pid = _start_fork( +start_func = _start_proc if returnproc else _start_fork + +pid = start_func( _exec_wrapper, args=( binary, @@ -652,6 +662,10 @@ def spawn( close_fds=close_fds, ) +
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/
commit: 190e8ad1a238d20d782235dd1faa6b00d1b3fd4a
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 3 21:24:49 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 7 00:55:45 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=190e8ad1
doebuild: Add returnproc parameter and deprecate returnpid
Raise NotImplementedError if returnproc is enabled for anything
other than the "depend" phase, since corresponding returnpid
support has long been deprecated.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py | 4 +--
lib/portage/package/ebuild/doebuild.py | 47 ++
2 files changed, 39 insertions(+), 12 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index a7c9650d74..f4f685e81c 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -124,7 +124,7 @@ class EbuildMetadataPhase(SubProcess):
mydbapi=self.portdb,
tree="porttree",
fd_pipes=fd_pipes,
-returnpid=True,
+returnproc=True,
)
settings.pop("PORTAGE_PIPE_FD", None)
@@ -137,7 +137,7 @@ class EbuildMetadataPhase(SubProcess):
self._async_wait()
return
-self._proc = portage.process.Process(retval[0])
+self._proc = retval
def _output_handler(self):
while True:
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index e10b884e08..1c89af5ac8 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -19,6 +19,7 @@ import sys
import tempfile
from textwrap import wrap
import time
+from typing import Union
import warnings
import zlib
@@ -246,14 +247,21 @@ def _doebuild_spawn(phase, settings, actionmap=None,
**kwargs):
def _spawn_phase(
-phase, settings, actionmap=None, returnpid=False, logfile=None, **kwargs
+phase,
+settings,
+actionmap=None,
+returnpid=False,
+returnproc=False,
+logfile=None,
+**kwargs,
):
-if returnpid:
+if returnproc or returnpid:
return _doebuild_spawn(
phase,
settings,
actionmap=actionmap,
returnpid=returnpid,
+returnproc=returnproc,
logfile=logfile,
**kwargs,
)
@@ -725,7 +733,8 @@ def doebuild(
prev_mtimes=None,
fd_pipes=None,
returnpid=False,
-):
+returnproc=False,
+) -> Union[int, portage.process.MultiprocessingProcess, list[int]]:
"""
Wrapper function that invokes specific ebuild phases through the spawning
of ebuild.sh
@@ -762,9 +771,15 @@ def doebuild(
for example.
@type fd_pipes: Dictionary
@param returnpid: Return a list of process IDs for a successful spawn, or
-an integer value if spawn is unsuccessful. NOTE: This requires the
-caller clean up all returned PIDs.
+an integer value if spawn is unsuccessful. This parameter is
supported
+supported only when mydo is "depend". NOTE: This requires the
caller clean
+up all returned PIDs.
@type returnpid: Boolean
+@param returnproc: Return a MultiprocessingProcess instance for a
successful spawn, or
+an integer value if spawn is unsuccessful. This parameter is
supported
+supported only when mydo is "depend". NOTE: This requires the
caller to
+asynchronously wait for the MultiprocessingProcess instance.
+@type returnproc: Boolean
@rtype: Boolean
@return:
1. 0 for success
@@ -867,17 +882,25 @@ def doebuild(
writemsg("\n", noiselevel=-1)
return 1
-if returnpid and mydo != "depend":
+if (returnproc or returnpid) and mydo != "depend":
# This case is not supported, since it bypasses the EbuildPhase class
# which implements important functionality (including post phase hooks
# and IPC for things like best/has_version and die).
+if returnproc:
+raise NotImplementedError(f"returnproc not implemented for phase
{mydo}")
warnings.warn(
"portage.doebuild() called "
"with returnpid parameter enabled. This usage will "
"not be supported in the future.",
-DeprecationWarning,
+UserWarning,
stacklevel=2,
)
+elif returnpid:
+warnings.warn(
+"The portage.doebuild() returnpid parameter is deprecated and
replaced by returnproc",
+UserWarning,
+stacklevel=1,
+)
if mydo == "fetchall":
fetchall = 1
@@ -1027,10 +1050,14 @@ def doebuild(
# get possible slot information from the deps file
if mydo == "depend":
-if not returnpid:
-raise
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
commit: e8b31c86eaed645a8740fb2844e2935aee161e43
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 5 05:55:11 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 6 01:30:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e8b31c86
ForkProcess: Prevent redundant pipe and set_term_size recursion
When process.spawn is updated to call ForkProcess
for bug 916566, it needs to avoid recursion via
set_term_size.
Bug: https://bugs.gentoo.org/916566
Bug: https://bugs.gentoo.org/923750
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/SpawnProcess.py| 31 ---
lib/portage/util/_async/ForkProcess.py | 28 +++-
2 files changed, 47 insertions(+), 12 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 7f4a23892b..716e94d665 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -41,7 +41,7 @@ class SpawnProcess(SubProcess):
)
__slots__ = (
-("args", "log_filter_file")
+("args", "create_pipe", "log_filter_file")
+ _spawn_kwarg_names
+ (
"_main_task",
@@ -60,15 +60,30 @@ class SpawnProcess(SubProcess):
else:
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
+log_file_path = None
if fd_pipes or self.logfile or not self.background:
-master_fd, slave_fd = self._pipe(fd_pipes)
+if self.create_pipe is not False:
+master_fd, slave_fd = self._pipe(fd_pipes)
-can_log = self._can_log(slave_fd)
-if can_log:
-log_file_path = self.logfile
+can_log = self._can_log(slave_fd)
+if can_log:
+log_file_path = self.logfile
else:
-log_file_path = None
+if self.logfile:
+raise NotImplementedError(
+"logfile conflicts with create_pipe=False"
+)
+# When called via process.spawn and ForkProcess._start,
+# SpawnProcess will have created a pipe earlier, so it
+# would be redundant to do it here (it could also trigger
+# spawn recursion via set_term_size as in bug 923750).
+# Use /dev/null for master_fd, triggering early return
+# of _main, followed by _async_waitpid.
+# TODO: Optimize away the need for master_fd here.
+master_fd = os.open(os.devnull, os.O_RDONLY)
+slave_fd = None
+can_log = False
null_input = None
if not self.background or 0 in fd_pipes:
@@ -97,7 +112,9 @@ class SpawnProcess(SubProcess):
fd_pipes_orig = fd_pipes.copy()
-if log_file_path is not None or self.background:
+if slave_fd is None:
+pass
+elif log_file_path is not None or self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index 3acbe34fc6..cb240d0712 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -75,12 +75,29 @@ class ForkProcess(SpawnProcess):
self.fd_pipes.setdefault(0, portage._get_stdin().fileno())
self.fd_pipes.setdefault(1, sys.__stdout__.fileno())
self.fd_pipes.setdefault(2, sys.__stderr__.fileno())
-stdout_fd = os.dup(self.fd_pipes[1])
+if self.create_pipe is not False:
+stdout_fd = os.dup(self.fd_pipes[1])
if self._HAVE_SEND_HANDLE:
-master_fd, slave_fd = self._pipe(self.fd_pipes)
-self.fd_pipes[1] = slave_fd
-self.fd_pipes[2] = slave_fd
+if self.create_pipe is not False:
+master_fd, slave_fd = self._pipe(self.fd_pipes)
+self.fd_pipes[1] = slave_fd
+self.fd_pipes[2] = slave_fd
+else:
+if self.logfile:
+raise NotImplementedError(
+"logfile conflicts with create_pipe=False"
+)
+# When called via process.spawn, SpawnProcess
+# will have created a pipe earlier, so it would be
+# redundant to do it here (it could also trigger spawn
+# recursion via set_term_size as in bug 923750). Use
+# /dev/null for master_fd, triggering early return
+# of _main, followed by _async_waitpid.
+# TODO: Optimize away the need for master_fd here.
+master_fd = os.open(os.devnull, os.O_RDONLY)
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/
commit: 0e4378fd18bcefde4472c7ae65e99578cf47dce6
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 4 07:59:26 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 4 08:00:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0e4378fd
Revert "doebuild: Add returnproc parameter and deprecate returnpid"
This reverts commit 6aeee45d9b88d7dc8c450ba86975e4b017db203c.
It triggered a "Bad file descriptor" during the instprep
phase as reported in bug 923755.
Bug: https://bugs.gentoo.org/923755
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py | 5 ++--
lib/portage/package/ebuild/doebuild.py | 47 --
2 files changed, 12 insertions(+), 40 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index b8164f2b16..a7c9650d74 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -108,7 +108,6 @@ class EbuildMetadataPhase(SubProcess):
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
)
-os.set_inheritable(slave_fd, True)
fd_pipes[slave_fd] = slave_fd
settings["PORTAGE_PIPE_FD"] = str(slave_fd)
@@ -125,7 +124,7 @@ class EbuildMetadataPhase(SubProcess):
mydbapi=self.portdb,
tree="porttree",
fd_pipes=fd_pipes,
-returnproc=True,
+returnpid=True,
)
settings.pop("PORTAGE_PIPE_FD", None)
@@ -138,7 +137,7 @@ class EbuildMetadataPhase(SubProcess):
self._async_wait()
return
-self._proc = retval
+self._proc = portage.process.Process(retval[0])
def _output_handler(self):
while True:
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 59a2b86a38..e10b884e08 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -19,7 +19,6 @@ import sys
import tempfile
from textwrap import wrap
import time
-from typing import Union
import warnings
import zlib
@@ -247,21 +246,14 @@ def _doebuild_spawn(phase, settings, actionmap=None,
**kwargs):
def _spawn_phase(
-phase,
-settings,
-actionmap=None,
-returnpid=False,
-returnproc=False,
-logfile=None,
-**kwargs,
+phase, settings, actionmap=None, returnpid=False, logfile=None, **kwargs
):
-if returnproc or returnpid:
+if returnpid:
return _doebuild_spawn(
phase,
settings,
actionmap=actionmap,
returnpid=returnpid,
-returnproc=returnproc,
logfile=logfile,
**kwargs,
)
@@ -733,8 +725,7 @@ def doebuild(
prev_mtimes=None,
fd_pipes=None,
returnpid=False,
-returnproc=False,
-) -> Union[int, portage.process.MultiprocessingProcess, list[int]]:
+):
"""
Wrapper function that invokes specific ebuild phases through the spawning
of ebuild.sh
@@ -771,15 +762,9 @@ def doebuild(
for example.
@type fd_pipes: Dictionary
@param returnpid: Return a list of process IDs for a successful spawn, or
-an integer value if spawn is unsuccessful. This parameter is
supported
-supported only when mydo is "depend". NOTE: This requires the
caller clean
-up all returned PIDs.
+an integer value if spawn is unsuccessful. NOTE: This requires the
+caller clean up all returned PIDs.
@type returnpid: Boolean
-@param returnproc: Return a MultiprocessingProcess instance for a
successful spawn, or
-an integer value if spawn is unsuccessful. This parameter is
supported
-supported only when mydo is "depend". NOTE: This requires the
caller to
-asynchronously wait for the MultiprocessingProcess instance.
-@type returnproc: Boolean
@rtype: Boolean
@return:
1. 0 for success
@@ -882,25 +867,17 @@ def doebuild(
writemsg("\n", noiselevel=-1)
return 1
-if (returnproc or returnpid) and mydo != "depend":
+if returnpid and mydo != "depend":
# This case is not supported, since it bypasses the EbuildPhase class
# which implements important functionality (including post phase hooks
# and IPC for things like best/has_version and die).
-if returnproc:
-raise NotImplementedError(f"returnproc not implemented for phase
{mydo}")
warnings.warn(
"portage.doebuild() called "
"with returnpid parameter enabled. This usage will "
"not be supported in the future.",
-UserWarning,
+DeprecationWarning,
stacklevel=2,
)
-elif returnpid:
-warnings.warn(
-"The portage.doebuild() returnpid paramenter is deprecated and
replaced by returnproc",
-
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/package/ebuild/
commit: 6aeee45d9b88d7dc8c450ba86975e4b017db203c
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 3 21:24:49 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 3 22:40:34 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6aeee45d
doebuild: Add returnproc parameter and deprecate returnpid
Raise NotImplementedError if returnproc is enabled for anything
other than the "depend" phase, since corresponding returnpid
support has long been deprecated.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py | 5 ++--
lib/portage/package/ebuild/doebuild.py | 47 ++
2 files changed, 40 insertions(+), 12 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index a7c9650d74..b8164f2b16 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -108,6 +108,7 @@ class EbuildMetadataPhase(SubProcess):
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
)
+os.set_inheritable(slave_fd, True)
fd_pipes[slave_fd] = slave_fd
settings["PORTAGE_PIPE_FD"] = str(slave_fd)
@@ -124,7 +125,7 @@ class EbuildMetadataPhase(SubProcess):
mydbapi=self.portdb,
tree="porttree",
fd_pipes=fd_pipes,
-returnpid=True,
+returnproc=True,
)
settings.pop("PORTAGE_PIPE_FD", None)
@@ -137,7 +138,7 @@ class EbuildMetadataPhase(SubProcess):
self._async_wait()
return
-self._proc = portage.process.Process(retval[0])
+self._proc = retval
def _output_handler(self):
while True:
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index e10b884e08..59a2b86a38 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -19,6 +19,7 @@ import sys
import tempfile
from textwrap import wrap
import time
+from typing import Union
import warnings
import zlib
@@ -246,14 +247,21 @@ def _doebuild_spawn(phase, settings, actionmap=None,
**kwargs):
def _spawn_phase(
-phase, settings, actionmap=None, returnpid=False, logfile=None, **kwargs
+phase,
+settings,
+actionmap=None,
+returnpid=False,
+returnproc=False,
+logfile=None,
+**kwargs,
):
-if returnpid:
+if returnproc or returnpid:
return _doebuild_spawn(
phase,
settings,
actionmap=actionmap,
returnpid=returnpid,
+returnproc=returnproc,
logfile=logfile,
**kwargs,
)
@@ -725,7 +733,8 @@ def doebuild(
prev_mtimes=None,
fd_pipes=None,
returnpid=False,
-):
+returnproc=False,
+) -> Union[int, portage.process.MultiprocessingProcess, list[int]]:
"""
Wrapper function that invokes specific ebuild phases through the spawning
of ebuild.sh
@@ -762,9 +771,15 @@ def doebuild(
for example.
@type fd_pipes: Dictionary
@param returnpid: Return a list of process IDs for a successful spawn, or
-an integer value if spawn is unsuccessful. NOTE: This requires the
-caller clean up all returned PIDs.
+an integer value if spawn is unsuccessful. This parameter is
supported
+supported only when mydo is "depend". NOTE: This requires the
caller clean
+up all returned PIDs.
@type returnpid: Boolean
+@param returnproc: Return a MultiprocessingProcess instance for a
successful spawn, or
+an integer value if spawn is unsuccessful. This parameter is
supported
+supported only when mydo is "depend". NOTE: This requires the
caller to
+asynchronously wait for the MultiprocessingProcess instance.
+@type returnproc: Boolean
@rtype: Boolean
@return:
1. 0 for success
@@ -867,17 +882,25 @@ def doebuild(
writemsg("\n", noiselevel=-1)
return 1
-if returnpid and mydo != "depend":
+if (returnproc or returnpid) and mydo != "depend":
# This case is not supported, since it bypasses the EbuildPhase class
# which implements important functionality (including post phase hooks
# and IPC for things like best/has_version and die).
+if returnproc:
+raise NotImplementedError(f"returnproc not implemented for phase
{mydo}")
warnings.warn(
"portage.doebuild() called "
"with returnpid parameter enabled. This usage will "
"not be supported in the future.",
-DeprecationWarning,
+UserWarning,
stacklevel=2,
)
+elif returnpid:
+warnings.warn(
+"The portage.doebuild() returnpid paramenter is deprecated and
replaced by returnproc",
+UserWarning,
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/, lib/portage/tests/ebuild/
commit: 305612d1b04aa06d3d1a1c8b51d046a644742fd5 Author: Zac Medico gentoo org> AuthorDate: Sat Feb 3 21:27:45 2024 + Commit: Zac Medico gentoo org> CommitDate: Sat Feb 3 21:27:45 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=305612d1 process.spawn: Use multiprocessing.Process for returnproc Use multiprocessing.Process for returnproc, so that fork will stop being used when python makes "spawn" the default multiprocessing start method. Continue to use _start_fork when returnproc is not enabled, for backward compatibility. Ultimately, it can be removed at the same time as the returnpid parameter. Bug: https://bugs.gentoo.org/916566 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/SpawnProcess.py| 4 +- lib/portage/process.py | 72 ++ lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 6 +- 3 files changed, 67 insertions(+), 15 deletions(-) diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py index 7f4a23892b..5e582e3322 100644 --- a/lib/_emerge/SpawnProcess.py +++ b/lib/_emerge/SpawnProcess.py @@ -224,7 +224,9 @@ class SpawnProcess(SubProcess): got_pty, master_fd, slave_fd = _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) -def _spawn(self, args: list[str], **kwargs) -> portage.process.Process: +def _spawn( +self, args: list[str], **kwargs +) -> portage.process.MultiprocessingProcess: spawn_func = portage.process.spawn if self._selinux_type is not None: diff --git a/lib/portage/process.py b/lib/portage/process.py index 01426179d7..d64ffa924f 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -19,7 +19,7 @@ import warnings from dataclasses import dataclass from functools import lru_cache -from typing import Any, Optional, Callable +from typing import Any, Optional, Callable, Union from portage import os from portage import _encodings @@ -28,6 +28,7 @@ import portage portage.proxy.lazyimport.lazyimport( globals(), +"portage.util._async.ForkProcess:ForkProcess", "portage.util._eventloop.global_event_loop:global_event_loop", "portage.util.futures:asyncio", "portage.util:dump_traceback,writemsg,writemsg_level", @@ -296,12 +297,19 @@ class AbstractProcess: class Process(AbstractProcess): """ -An object that wraps OS processes created by spawn. -In the future, spawn will return objects of a different type -but with a compatible interface to this class, in order -to encapsulate implementation-dependent objects like -multiprocessing.Process which are designed to manage -the process lifecycle and need to persist until it exits. +An object that wraps OS processes which do not have an +associated multiprocessing.Process instance. Ultimately, +we need to stop using os.fork() to create these processes +because it is unsafe for threaded processes as discussed +in https://github.com/python/cpython/issues/84559. + +Note that if subprocess.Popen is used without pass_fds +or preexec_fn parameters, then it avoids using os.fork() +by instead using posix_spawn. This approach is not used +by spawn because it needs to execute python code prior +to exec, so it instead uses multiprocessing.Process, +which only uses os.fork() when the multiprocessing start +method is fork. """ def __init__(self, pid: int): @@ -461,7 +469,7 @@ def spawn( unshare_mount=False, unshare_pid=False, warn_on_large_env=False, -): +) -> Union[int, MultiprocessingProcess, list[int]]: """ Spawns a given command. @@ -479,8 +487,8 @@ def spawn( @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean -@param returnproc: Return a Process object for a successful spawn (conflicts with logfile parameter). -NOTE: This requires the caller to asynchronously wait for the Process. +@param returnproc: Return a MultiprocessingProcess instance (conflicts with logfile parameter). +NOTE: This requires the caller to asynchronously wait for the MultiprocessingProcess instance. @type returnproc: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @@ -623,7 +631,9 @@ def spawn( # fork, so that the result is cached in the main process. bool(groups) -pid = _start_fork( +start_func = _start_proc if returnproc else _start_fork + +pid = start_func( _exec_wrapper, args=( binary, @@ -649,6 +659,10 @@ def spawn( close_fds=close_fds, ) +if returnproc: +# _start_proc returns a MultiprocessingProcess instance. +return pid + if not isinstance(pid, int): raise
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: da5b18a418233e289229d3d124f6352e531c9ffb
Author: Zac Medico gentoo org>
AuthorDate: Wed Jan 17 12:49:32 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Jan 17 12:50:46 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=da5b18a4
depgraph: Fix SyntaxWarning: 'str' object is not callable
Fixes: a5078e577493 ("emerge: backtrack consistently regardless of --fetchonly")
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 7491d970c3..1cbbbaf815 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -3794,7 +3794,8 @@ class depgraph:
except KeyError:
optional_msg = " ({} previously removed from graph)"
warnings.warn(
-f"_eliminate_rebuilds inconsistency: parent
priorities missing for {parent} -> {pkg} edge"(
+f"_eliminate_rebuilds inconsistency: parent
priorities missing for {parent} -> {pkg} edge"
++ (
optional_msg.format("parent and child")
if parent in removed and pkg in removed
else optional_msg.format("parent")
@@ -3815,7 +3816,8 @@ class depgraph:
except KeyError:
optional_msg = " ({} previously removed from graph)"
warnings.warn(
-f"_eliminate_rebuilds inconsistency: parent
priorities missing for {pkg} -> {child} edge"(
+f"_eliminate_rebuilds inconsistency: parent
priorities missing for {pkg} -> {child} edge"
++ (
optional_msg.format("parent and child")
if pkg in removed and child in removed
else optional_msg.format("parent")
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, man/
commit: a5078e5774932103d4ad367de4c6bf130a6da34f
Author: Zac Medico gentoo org>
AuthorDate: Mon Jan 15 23:10:03 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Jan 16 16:01:53 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a5078e57
emerge: backtrack consistently regardless of --fetchonly
Make the _accept_blocker_conflicts method always return
True when backtracking is enabled, so that backtracking
results will be identical regardless of options that
_accept_blocker_conflicts treats specially. This way,
conflicts will only be accepted when backtracking is
disabled or all backtracking tries have been exhausted.
Make --nodeps imply --backtrack=0, since backtracking
is only useful with dependencies.
Make _eliminate_rebuilds return early if there are
slot conflicts, since the merge list is invalid anyway,
and its possible that state inconsistencies could
trigger unexpected exceptions as in bug 922038. Make
the KeyError from bug 922038 a warning, and include
relevant information to help trace the inconsistency
back to the _eliminate_rebuilds method or some other
source like _solve_non_slot_operator_slot_conflicts.
Bug: https://bugs.gentoo.org/161422
Bug: https://bugs.gentoo.org/607252
Bug: https://bugs.gentoo.org/675748
Bug: https://bugs.gentoo.org/922038
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py | 63 +
man/emerge.1| 5 ++--
2 files changed, 61 insertions(+), 7 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index b859e68224..7491d970c3 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -3674,9 +3674,19 @@ class depgraph:
careful to obey the user's wishes if they have explicitly requested
for a package to be rebuilt or reinstalled for some reason.
"""
-if "empty" in self._dynamic_config.myparams:
+# Skip for slot conflicts since the merge list is not valid
+# anyway, and possible state inconsistencies can trigger
+# unexpected exceptions as in bug 922038.
+if "empty" in self._dynamic_config.myparams or any(
+self._dynamic_config._package_tracker.slot_conflicts()
+):
return False
+# Track packages that we remove from the graph during
+# this method call, in order to help trace any detected
+# inconsistency back to this method or some other source
+# such as _solve_non_slot_operator_slot_conflicts.
+removed = []
modified = False
selective = "selective" in self._dynamic_config.myparams
for root, atom in
self._dynamic_config._slot_operator_replace_installed:
@@ -3777,11 +3787,45 @@ class depgraph:
modified = True
parent_atoms = []
for parent, parent_atom in
self._dynamic_config._parent_atoms[pkg]:
-priorities =
self._dynamic_config.digraph.nodes[pkg][1][parent][:]
+try:
+priorities =
self._dynamic_config.digraph.nodes[pkg][1][parent][
+:
+]
+except KeyError:
+optional_msg = " ({} previously removed from graph)"
+warnings.warn(
+f"_eliminate_rebuilds inconsistency: parent
priorities missing for {parent} -> {pkg} edge"(
+optional_msg.format("parent and child")
+if parent in removed and pkg in removed
+else optional_msg.format("parent")
+if parent in removed
+else optional_msg.format("child")
+if pkg in removed
+else ""
+)
+)
+priorities = []
parent_atoms.append((parent, parent_atom, priorities))
child_parents = {}
for child in self._dynamic_config.digraph.child_nodes(pkg):
-priorities =
self._dynamic_config.digraph.nodes[child][1][pkg][:]
+try:
+priorities =
self._dynamic_config.digraph.nodes[child][1][pkg][
+:
+]
+except KeyError:
+optional_msg = " ({} previously removed from graph)"
+warnings.warn(
+f"_eliminate_rebuilds inconsistency: parent
priorities missing for {pkg} -> {child} edge"(
+optional_msg.format("parent and child")
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/resolver/, lib/_emerge/
commit: 7402c4f2b9eef426134aca6a6ba71bbbd6389dac
Author: Sam James gentoo org>
AuthorDate: Fri Jan 5 11:25:51 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Jan 5 11:25:51 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7402c4f2
lib/_emerge: codespell fixes
Signed-off-by: Sam James gentoo.org>
lib/_emerge/depgraph.py | 2 +-
lib/_emerge/resolver/circular_dependency.py | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index efe084a780..a2865cad23 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -11927,7 +11927,7 @@ def _resume_depgraph(
) or parent_node.operation not in ("merge", "nomerge"):
continue
# We need to traverse all priorities here, in order to
-# ensure that a package with an unsatisfied depenedency
+# ensure that a package with an unsatisfied dependency
# won't get pulled in, even indirectly via a soft
# dependency.
unsatisfied_stack.append((parent_node, atom))
diff --git a/lib/_emerge/resolver/circular_dependency.py
b/lib/_emerge/resolver/circular_dependency.py
index cd612e3130..6c21423083 100644
--- a/lib/_emerge/resolver/circular_dependency.py
+++ b/lib/_emerge/resolver/circular_dependency.py
@@ -189,7 +189,7 @@ class circular_dependency_handler:
if len(affecting_use) > self.MAX_AFFECTING_USE:
# Limit the number of combinations explored (bug #555698).
-# First, discard irrelevent flags that are not enabled.
+# First, discard irrelevant flags that are not enabled.
# Since extract_affecting_use doesn't distinguish between
# positive and negative effects (flag? vs. !flag?), assume
# a positive relationship.
@@ -254,8 +254,8 @@ class circular_dependency_handler:
continue
# Check if a USE change conflicts with use requirements of the
parents.
-# If a requiremnet is hard, ignore the suggestion.
-# If the requirment is conditional, warn the user that other
changes might be needed.
+# If a requirement is hard, ignore the suggestion.
+# If the requirement is conditional, warn the user that other
changes might be needed.
followup_change = False
parent_parent_atoms =
self.depgraph._dynamic_config._parent_atoms.get(
changed_parent
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 0ff49114cec79edce723da1190087a41699f6b2f
Author: Zac Medico gentoo org>
AuthorDate: Thu Jan 4 05:12:24 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Jan 4 06:13:25 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0ff49114
binarytree: Handle inject failures
Capture stdout and stderr for logging purposes during binarytree
inject calls, and use the inject return value to report success
or failure.
For failures prior to pkg_pretend, use an eerror/elog message
to indicate that the binary package is not usable. Move
corresponding elog_process call to a finally block so that it
is called for all pkg_pretend failures.
Bug: https://bugs.gentoo.org/921327
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/Binpkg.py | 34 ++
lib/_emerge/BinpkgPrefetcher.py | 36 +---
lib/_emerge/Scheduler.py| 36 +++-
3 files changed, 82 insertions(+), 24 deletions(-)
diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index 9b1036538a..299ae7fbc9 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -1,6 +1,8 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import io
+import sys
import functools
import _emerge.emergelog
from _emerge.EbuildPhase import EbuildPhase
@@ -244,12 +246,36 @@ class Binpkg(CompositeTask):
pkg_count = self.pkg_count
if self._fetched_pkg:
-pkg_path = self._bintree.getname(
-self._bintree.inject(
+stdout_orig = sys.stdout
+stderr_orig = sys.stderr
+out = io.StringIO()
+try:
+sys.stdout = out
+sys.stderr = out
+
+injected_pkg = self._bintree.inject(
pkg.cpv,
current_pkg_path=self._fetched_pkg,
allocated_pkg_path=self._pkg_allocated_path,
-),
+)
+finally:
+sys.stdout = stdout_orig
+sys.stderr = stderr_orig
+
+output_value = out.getvalue()
+if output_value:
+self.scheduler.output(
+output_value,
+log_path=self.settings.get("PORTAGE_LOG_FILE"),
+background=self.background,
+)
+
+if injected_pkg is None:
+self._async_unlock_builddir(returncode=1)
+return
+
+pkg_path = self._bintree.getname(
+injected_pkg,
allocate_new=False,
)
else:
diff --git a/lib/_emerge/BinpkgPrefetcher.py b/lib/_emerge/BinpkgPrefetcher.py
index 37dbe0a40f..ed68d2852c 100644
--- a/lib/_emerge/BinpkgPrefetcher.py
+++ b/lib/_emerge/BinpkgPrefetcher.py
@@ -1,6 +1,9 @@
-# Copyright 1999-2009 Gentoo Foundation
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import io
+import sys
+
from _emerge.BinpkgFetcher import BinpkgFetcher
from _emerge.CompositeTask import CompositeTask
from _emerge.BinpkgVerifier import BinpkgVerifier
@@ -45,12 +48,31 @@ class BinpkgPrefetcher(CompositeTask):
self.wait()
return
-self._bintree.inject(
-self.pkg.cpv,
-current_pkg_path=self.pkg_path,
-allocated_pkg_path=self.pkg_allocated_path,
-)
+stdout_orig = sys.stdout
+stderr_orig = sys.stderr
+out = io.StringIO()
+try:
+sys.stdout = out
+sys.stderr = out
+
+injected_pkg = self._bintree.inject(
+self.pkg.cpv,
+current_pkg_path=self.pkg_path,
+allocated_pkg_path=self.pkg_allocated_path,
+)
+
+finally:
+sys.stdout = stdout_orig
+sys.stderr = stderr_orig
+
+output_value = out.getvalue()
+if output_value:
+self.scheduler.output(
+output_value,
+log_path=self.scheduler.fetch.log_file,
+background=self.background,
+)
self._current_task = None
-self.returncode = os.EX_OK
+self.returncode = 1 if injected_pkg is None else os.EX_OK
self.wait()
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index 620d513511..ae01214d32 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from collections import deque
@@ -984,11 +984,19 @@ class Scheduler(PollScheduler):
current_task = None
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: d8285bad5815f0f2db64657dd245efc3fe948210 Author: Sam James gentoo org> AuthorDate: Wed Jan 3 06:08:53 2024 + Commit: Sam James gentoo org> CommitDate: Wed Jan 3 19:59:51 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8285bad BinpkgFetcher: preserve mtime The FileCopier from 1db44d18578a7aee58449cb97e1991cb06c915c3 doesn't preserve its timestamp so there's a mismatch between the expected mtime in the index and the binpkg. Bug: https://bugs.gentoo.org/921208 Thanks-to: Zac Medico gentoo.org> Signed-off-by: Sam James gentoo.org> lib/_emerge/BinpkgFetcher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index 01b2bae637..a1524dc009 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -106,7 +106,8 @@ class BinpkgFetcher(CompositeTask): finally: if copier.isAlive(): copier.cancel() - +if copier.returncode == os.EX_OK: +fetcher.sync_timestamp() else: fetcher.start() try:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 89df7574a355a245e19ba297c3685997eec6bbbe
Author: Zac Medico gentoo org>
AuthorDate: Sat Dec 30 18:21:08 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Dec 30 18:27:21 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=89df7574
EbuildBinpkg: Handle binarytree inject failure
Capture stdout and stderr for logging during
binarytree inject, and set returncode to 1 when
inject fails and returns None.
Bug: https://bugs.gentoo.org/921089
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildBinpkg.py | 37 ++---
1 file changed, 30 insertions(+), 7 deletions(-)
diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py
index cb01f73d73..97b69ae018 100644
--- a/lib/_emerge/EbuildBinpkg.py
+++ b/lib/_emerge/EbuildBinpkg.py
@@ -1,6 +1,9 @@
-# Copyright 1999-2012 Gentoo Foundation
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import io
+import sys
+
from _emerge.CompositeTask import CompositeTask
from _emerge.EbuildPhase import EbuildPhase
@@ -55,14 +58,34 @@ class EbuildBinpkg(CompositeTask):
pkg = self.pkg
bintree = pkg.root_config.trees["bintree"]
-self._binpkg_info = bintree.inject(
-pkg.cpv,
-current_pkg_path=self._binpkg_tmpfile,
-allocated_pkg_path=self.pkg_allocated_path,
-)
+
+stdout_orig = sys.stdout
+stderr_orig = sys.stderr
+out = io.StringIO()
+try:
+sys.stdout = out
+sys.stderr = out
+
+self._binpkg_info = bintree.inject(
+pkg.cpv,
+current_pkg_path=self._binpkg_tmpfile,
+allocated_pkg_path=self.pkg_allocated_path,
+)
+
+finally:
+sys.stdout = stdout_orig
+sys.stderr = stderr_orig
+
+output_value = out.getvalue()
+if output_value:
+self.scheduler.output(
+output_value,
+log_path=self.settings.get("PORTAGE_LOG_FILE"),
+background=self.background,
+)
self._current_task = None
-self.returncode = os.EX_OK
+self.returncode = 1 if self._binpkg_info is None else os.EX_OK
self.wait()
def get_binpkg_info(self):
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 8b167b0adffac5bdf1d7e43f2ca3da5f8d604b33
Author: Eli Schwartz gmail com>
AuthorDate: Wed Dec 27 02:12:03 2023 +
Commit: Sam James gentoo org>
CommitDate: Wed Dec 27 13:30:19 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8b167b0a
emerge: enable "avoid spamming too much info about unused binpkgs" again
In commit a6853d5493b7bed37e60c2e3b7536b209500ba3f this code in
_show_ignored_binaries_respect_use was refactored to handle pkg.root.
But it was refactored incorrectly -- the storage/lookup key started off
as:
```
seen[pkg.cpv]
```
and was refactored so the storage key was:
```
seen[pkg.root][pkg.cpv]
```
and the lookup key was
```
seen[(pkg.root, pkg.cpv)]
```
As a result, we never detected a previously seen USE flags combo, and
the logic to avoid spamming too much info was a no-op; the info was
spammed, instead.
Note that even though we have more code than before this patch, we
actually have less code. The black code formatter decided that since the
line length was decreased, this entire code block should be reformatted,
murdering the diff view and dividing less code across *more* lines. This
is not my fault and I refuse to be held accountable for it -- if you git
blame this and do not know what happened, understand that it happens
despite my objections.
Signed-off-by: Eli Schwartz gmail.com>
Closes: https://github.com/gentoo/portage/pull/1219
Signed-off-by: Sam James gentoo.org>
lib/_emerge/depgraph.py | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 6ee4471bbe..efe084a780 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1256,9 +1256,10 @@ class depgraph:
# We don't want to list the same USE flags for multiple build IDs
seen.setdefault(pkg.root, dict())
-if (pkg.root, pkg.cpv) not in seen or flag_display not in
seen[pkg.root][
-pkg.cpv
-]:
+if (
+pkg.cpv not in seen[pkg.root]
+or flag_display not in seen[pkg.root][pkg.cpv]
+):
seen[pkg.root].setdefault(pkg.cpv, set()).add(flag_display)
# The user can paste this line into package.use
messages.append(f"={pkg.cpv} {flag_display}")
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: fbd6909c24f51ad79fb313ddd7043b4a40de91a6
Author: Zac Medico gentoo org>
AuthorDate: Tue Dec 26 21:37:24 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Dec 26 22:01:51 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fbd6909c
Scheduler: Handle unpack_metadata SignatureException in _run_pkg_pretend
This will handle the InvalidSignature exception that triggered
bug 920258, allowing emerge --keep-going to skip the package
if possible. All of the gpg stderr output is dumped before the
exception is raised, as shown in bug 920258 comment #0.
Bug: https://bugs.gentoo.org/920258
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/Scheduler.py | 11 ++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index 4193ffdd03..620d513511 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -992,7 +992,16 @@ class Scheduler(PollScheduler):
infloc = os.path.join(build_dir_path, "build-info")
ensure_dirs(infloc)
-await bintree.dbapi.unpack_metadata(settings, infloc,
loop=loop)
+try:
+await bintree.dbapi.unpack_metadata(settings, infloc,
loop=loop)
+except portage.exception.SignatureException as e:
+writemsg(
+f"!!! Invalid binary package:
'{bintree.getname(x.cpv)}', {e}\n",
+noiselevel=-1,
+)
+failures += 1
+self._record_pkg_failure(x, settings, 1)
+continue
ebuild_path = os.path.join(infloc, x.pf + ".ebuild")
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: bf3d091de8702f0c95e5530d03c6e925008ee80a
Author: Zac Medico gentoo org>
AuthorDate: Sun Dec 24 05:12:55 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Dec 24 19:29:25 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bf3d091d
depclean: drop direct circular deps in merge order calculation
Drop direct circular deps in the depclean merge order calculation,
since it does not handle them well as shown by the test case
for bug 916135.
Bug: https://bugs.gentoo.org/916135
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/actions.py| 8 ++--
lib/portage/tests/resolver/test_depclean_order.py | 10 ++
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 86ba7f77a5..13bb75931c 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import collections
@@ -1642,7 +1642,11 @@ def _calc_depclean(settings, trees, ldpath_mtimes,
myopts, action, args_set, spi
if mypriority.runtime:
mypriority.runtime_slot_op = True
-graph.add(child_node, node, priority=mypriority)
+# Drop direct circular deps because the unmerge
order
+# calculation does not handle them well as
demonstrated
+# by the test case for bug 916135.
+if child_node is not node:
+graph.add(child_node, node,
priority=mypriority)
if debug:
writemsg_level("\nunmerge digraph:\n\n", noiselevel=-1,
level=logging.DEBUG)
diff --git a/lib/portage/tests/resolver/test_depclean_order.py
b/lib/portage/tests/resolver/test_depclean_order.py
index 867b1a54ca..23b5e755c3 100644
--- a/lib/portage/tests/resolver/test_depclean_order.py
+++ b/lib/portage/tests/resolver/test_depclean_order.py
@@ -1,8 +1,6 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import pytest
-
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
@@ -60,8 +58,12 @@ class SimpleDepcleanTestCase(TestCase):
finally:
playground.cleanup()
-@pytest.mark.xfail()
def testIDEPENDDepclean(self):
+"""
+Test for bug 916135, where a direct circular dependency caused
+the unmerge order to fail to account for IDEPEND.
+"""
+
ebuilds = {
"dev-util/A-1": {},
"dev-libs/B-1": {
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 1477124cc20c8ff3ca3be07efe98fafeb990 Author: Zac Medico gentoo org> AuthorDate: Mon Dec 18 18:35:14 2023 + Commit: Zac Medico gentoo org> CommitDate: Mon Dec 18 18:35:46 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1477124c Scheduler: Reset current_task in _run_pkg_pretend For completed tasks, reset current_task to None so that it does not cause the clean phase to execute with an invalid environment in the finally block. Bug: https://bugs.gentoo.org/920258 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/Scheduler.py | 4 1 file changed, 4 insertions(+) diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index 66eca4c650..4193ffdd03 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -930,6 +930,7 @@ class Scheduler(PollScheduler): current_task = clean_phase clean_phase.start() await clean_phase.async_wait() +current_task = None if x.built: tree = "bintree" @@ -981,6 +982,7 @@ class Scheduler(PollScheduler): self._record_pkg_failure(x, settings, verifier.returncode) continue +current_task = None if fetched: bintree.inject( x.cpv, @@ -1030,6 +1032,8 @@ class Scheduler(PollScheduler): current_task = pretend_phase pretend_phase.start() ret = await pretend_phase.async_wait() +# Leave current_task assigned in order to trigger clean +# on success in the below finally block. if ret != os.EX_OK: failures += 1 self._record_pkg_failure(x, settings, ret)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, /
commit: b7e35beaa4168b315ad3ae294b71706d24c795b0
Author: John Helmert III gentoo org>
AuthorDate: Thu Nov 16 03:45:14 2023 +
Commit: John Helmert III gentoo org>
CommitDate: Fri Dec 15 18:30:59 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7e35bea
depclean: return failure when no packages are selected for depclean
Based on the preceding writemsg, one might think that this function
could be used for more than depcleans, but there's no other reference
to it in the codebase.
With this patch:
$ sudo ./bin/emerge --debug --verbose --depclean foo; echo $?
myaction depclean
myopts {'--debug': True, '--binpkg-respect-use': 'y', '--quiet-build': 'y',
'--regex-search-auto': 'y', '--verbose': True}
[DEBUG] Using selector: EpollSelector
Signed-off-by: John Helmert III gentoo.org>
NEWS | 6 ++
lib/_emerge/actions.py | 2 +-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/NEWS b/NEWS
index 250b4399f2..e1e47d0a2e 100644
--- a/NEWS
+++ b/NEWS
@@ -6,6 +6,12 @@ Release notes take the form of the following optional
categories:
* Bug fixes
* Cleanups
+portage-3.0.59 (UNRELEASED)
+
+Features:
+* emerge: depclean now returns with failure if no packages are matched
+ (bug #917120)
+
portage-3.0.58 (2023-12-14)
--
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index ae8796531e..86ba7f77a5 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -843,7 +843,7 @@ def action_depclean(
)
if not matched_packages:
writemsg_level(f">>> No packages selected for removal by
{action}\n")
-return 0
+return 1
# The calculation is done in a separate function so that depgraph
# references go out of scope and the corresponding memory
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 69ce8a71cc806b4b333c2e707e7a9291c3d84664
Author: Sam James gentoo org>
AuthorDate: Sun Dec 10 22:13:09 2023 +
Commit: Sam James gentoo org>
CommitDate: Sun Dec 10 22:29:08 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=69ce8a71
_emerge: BinpkgVerifier: give better error message on stale binpkg index
portage-3.0.52 defaults to FEATURES="pkgdir-index-trusted" (see NEWS) which
has a few benefits, but means that manually editing PKGDIR without regenerating
the index with 'emaint binhost -f' will confuse Portage.
Give a better error message mentioning that command if we fail to fetch a binpkg
but bintree.dbapi.cpv_exists says it should exist.
Bug: https://bugs.gentoo.org/915474
Bug: https://bugs.gentoo.org/918597
Signed-off-by: Sam James gentoo.org>
lib/_emerge/BinpkgVerifier.py | 22 +-
1 file changed, 17 insertions(+), 5 deletions(-)
diff --git a/lib/_emerge/BinpkgVerifier.py b/lib/_emerge/BinpkgVerifier.py
index a7917453aa..7e044c6c43 100644
--- a/lib/_emerge/BinpkgVerifier.py
+++ b/lib/_emerge/BinpkgVerifier.py
@@ -41,11 +41,23 @@ class BinpkgVerifier(CompositeTask):
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
-self.scheduler.output(
-f"!!! Fetching Binary failed for '{self.pkg.cpv}'\n",
-log_path=self.logfile,
-background=self.background,
-)
+
+# We might end up here with FEATURES="pkgdir-index-trusted" if
+# binpkgs have been removed manually without refreshing the index.
+if bintree.dbapi.cpv_exists(self.pkg.cpv):
+self.scheduler.output(
+f"!!! Tried to use non-existent binary for
'{self.pkg.cpv}'\n"
++ f"!!! Likely caused by an outdated index. Run 'emaint
binhost -f'.\n",
+log_path=self.logfile,
+background=self.background,
+)
+else:
+self.scheduler.output(
+f"!!! Fetching Binary failed for '{self.pkg.cpv}'\n",
+log_path=self.logfile,
+background=self.background,
+)
+
self.returncode = 1
self._async_wait()
return
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/emerge/, lib/portage/tests/dep/, ...
commit: af550b8b5cb91f27b26d6800c3b4cdd2d86a46e6
Author: Sam James gentoo org>
AuthorDate: Mon Sep 4 15:38:18 2023 +
Commit: Sam James gentoo org>
CommitDate: Sun Dec 10 22:01:48 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=af550b8b
ebuild: inject implicit libc RDEPEND
Inject >=${LIBC_PROVIDER}-${VERSION_OF_LIBC_PROVIDER} into RDEPEND.
We already try to upgrade the virtual/libc provider and its deps aggressively
but that's not so helpful if there's a binpkg for, say, one of its deps
available
built against a newer glibc - which ends up breaking the system because our
glibc isn't new enough (symbol versioning).
This is a long-standing problem for binpkg users and one of the last
showstoppers
for being able to use them reliably.
Note that this applies to source installs too, although it matters less there;
it'll make downgrading libc a bit harder for people who want to do that, but
users
are already prevented from doing that (pkg_* check) for glibc, and I don't
really
see it as a bad thing to effectively propagate this to other libcs.
To fully solve the problem, we should arguably do at least one of the following:
1) Implicit >= on anything (not just libc) which installs ELF (at least a
PROVIDES);
2) Implement the suggestion in bug #753500 based on analysis of used versioned
symbols (*).
But libc is really the critical one and the one where things explode pretty
badly,
especially combined with us trying to Do The Right Thing for non-binpkg cases
(aggressively upgrading libc before anything else). The other cases don't matter
so much as they get upgraded later and by that point, the library is usually
done.
(It's not really clear if downgrading musl works and even if it is supported,
I'm not really sure it's a supported case at all, so I'm not bothering to carve
out an exception here. It'd make this far less elegant and I don't see any
benefit
to doing so.)
(*) util-linux is one of the examples I have in mind here as justification
for either point.
Bug: https://bugs.gentoo.org/753500
Bug: https://bugs.gentoo.org/913628
Signed-off-by: Sam James gentoo.org>
lib/_emerge/actions.py | 7 +-
lib/_emerge/depgraph.py | 22 +
lib/portage/dep/__init__.py | 23 +-
lib/portage/dep/libc.py | 83
lib/portage/dep/meson.build | 1 +
lib/portage/package/ebuild/doebuild.py | 56 ++-
lib/portage/tests/dep/meson.build| 1 +
lib/portage/tests/dep/test_libc.py | 81
lib/portage/tests/emerge/meson.build | 1 +
lib/portage/tests/emerge/test_actions.py | 23 +-
lib/portage/tests/emerge/test_libc_dep_inject.py | 551 +++
11 files changed, 831 insertions(+), 18 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index dbd9707a82..ae8796531e 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -41,6 +41,7 @@ from portage.dbapi._expand_new_virt import expand_new_virt
from portage.dbapi.IndexedPortdb import IndexedPortdb
from portage.dbapi.IndexedVardb import IndexedVardb
from portage.dep import Atom, _repo_separator, _slot_separator
+from portage.dep.libc import find_libc_deps
from portage.exception import (
InvalidAtom,
InvalidData,
@@ -2786,10 +2787,8 @@ def relative_profile_path(portdir, abs_profile):
def get_libc_version(vardb: portage.dbapi.vartree.vardbapi) -> list[str]:
libcver = []
-libclist = set()
-for atom in expand_new_virt(vardb, portage.const.LIBC_PACKAGE_ATOM):
-if not atom.blocker:
-libclist.update(vardb.match(atom))
+libclist = find_libc_deps(vardb, True)
+
if libclist:
for cpv in sorted(libclist):
libc_split = portage.catpkgsplit(cpv)[1:]
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 59c78c7354..4612ac2049 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -36,6 +36,7 @@ from portage.dep import (
match_from_list,
_repo_separator,
)
+from portage.dep.libc import find_libc_deps, strip_libc_deps
from portage.dep._slot_operator import ignore_built_slot_operator_deps,
strip_slots
from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use,
_get_eapi_attrs
from portage.exception import (
@@ -2984,6 +2985,19 @@ class depgraph:
else:
depvars = Package._runtime_keys
+eroot = pkg.root_config.settings["EROOT"]
+try:
+libc_deps = self._frozen_config._libc_deps_cache[eroot]
+except (AttributeError, KeyError) as e:
+if isinstance(e, AttributeError):
+self._frozen_config._libc_deps_cache = {}
+
+self._frozen_config._libc_deps_cache[eroot] = find_libc_deps(
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 1d856747ada48f8d32c033091b1156cc655efed3
Author: Zac Medico gentoo org>
AuthorDate: Wed Dec 6 06:05:46 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Dec 6 20:23:14 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1d856747
DepPriority{Normal,Satisfied}Range: weaken _ignore_runtime for cross root
When the dependency parent is for a cross root (ROOT != /) package,
weaken _ignore_runtime in order to tolerate runtime cycles that are
less problematic for cross root packages.
The included test case fails with this error without the fix:
* Error: circular dependencies:
(dev-libs/gmp-6.3.0:0/10.4::test_repo, binary scheduled for merge to
'/tmp/tmp25nwdjn7/cross_root/') depends on
(sys-devel/gcc-13.2.1_p20230826:0/0::test_repo, binary scheduled for merge to
'/tmp/tmp25nwdjn7/cross_root/') (runtime)
(dev-libs/gmp-6.3.0:0/10.4::test_repo, binary scheduled for merge to
'/tmp/tmp25nwdjn7/cross_root/') (runtime_slot_op)
It might be possible to break this cycle
by applying the following change:
- dev-libs/gmp-6.3.0 (Change USE: -cxx)
Bug: https://bugs.gentoo.org/919174
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/DepPriority.py | 4 +-
lib/_emerge/DepPriorityNormalRange.py | 4 +-
lib/_emerge/DepPrioritySatisfiedRange.py | 1 +
lib/_emerge/UnmergeDepPriority.py | 3 +-
lib/_emerge/depgraph.py| 46 --
lib/portage/tests/resolver/meson.build | 1 +
.../tests/resolver/test_cross_dep_priority.py | 164 +
7 files changed, 208 insertions(+), 15 deletions(-)
diff --git a/lib/_emerge/DepPriority.py b/lib/_emerge/DepPriority.py
index 99d38477e2..8d282b937a 100644
--- a/lib/_emerge/DepPriority.py
+++ b/lib/_emerge/DepPriority.py
@@ -1,11 +1,11 @@
-# Copyright 1999-2013 Gentoo Foundation
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.AbstractDepPriority import AbstractDepPriority
class DepPriority(AbstractDepPriority):
-__slots__ = ("satisfied", "optional", "ignored")
+__slots__ = ("cross", "ignored", "optional", "satisfied")
def __int__(self):
"""
diff --git a/lib/_emerge/DepPriorityNormalRange.py
b/lib/_emerge/DepPriorityNormalRange.py
index d7e4381b47..cb0e6c26b1 100644
--- a/lib/_emerge/DepPriorityNormalRange.py
+++ b/lib/_emerge/DepPriorityNormalRange.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2011 Gentoo Foundation
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.DepPriority import DepPriority
@@ -41,7 +41,7 @@ class DepPriorityNormalRange:
# to adjust this appropriately. But only build time dependencies
# are optional right now, so it's not an issue as-is.
return bool(
-not priority.runtime_slot_op
+not (priority.runtime_slot_op and not priority.cross)
and (priority.optional or not priority.buildtime)
)
diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py
b/lib/_emerge/DepPrioritySatisfiedRange.py
index 0d42e7613d..b3bc90c2ff 100644
--- a/lib/_emerge/DepPrioritySatisfiedRange.py
+++ b/lib/_emerge/DepPrioritySatisfiedRange.py
@@ -96,6 +96,7 @@ class DepPrioritySatisfiedRange:
(
(not priority.runtime_slot_op)
or (priority.satisfied and priority.runtime_slot_op)
+or priority.cross
)
and (priority.satisfied or priority.optional or not
priority.buildtime)
)
diff --git a/lib/_emerge/UnmergeDepPriority.py
b/lib/_emerge/UnmergeDepPriority.py
index ff81eff46f..d818bad1b8 100644
--- a/lib/_emerge/UnmergeDepPriority.py
+++ b/lib/_emerge/UnmergeDepPriority.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2013 Gentoo Foundation
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.AbstractDepPriority import AbstractDepPriority
@@ -6,6 +6,7 @@ from _emerge.AbstractDepPriority import AbstractDepPriority
class UnmergeDepPriority(AbstractDepPriority):
__slots__ = (
+"cross",
"ignored",
"optional",
"satisfied",
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 9b09701021..59c78c7354 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -3630,7 +3630,7 @@ class depgraph:
blocker=False,
depth=depth,
parent=pkg,
-priority=self._priority(runtime=True),
+priority=self._priority(cross=self._cross(pkg.root),
runtime=True),
root=pkg.root,
)
if not self._add_dep(dep, allow_unsatisfied=allow_unsatisfied):
@@ -3968,17 +3968,26 @@ class depgraph:
# _dep_disjunctive_stack first, so
[gentoo-commits] proj/portage:master commit in: lib/_emerge/resolver/
commit: 036a0785d2595defc55ef03e424ffdf739a108b1 Author: Zac Medico gentoo org> AuthorDate: Wed Dec 6 00:26:17 2023 + Commit: Zac Medico gentoo org> CommitDate: Wed Dec 6 16:06:00 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=036a0785 circular_dependency: Handle SonameAtom Treat soname deps is as unconditional for now. In some cases they can be avoided by a rebuild with changed USE, but ebuilds sometimes do not specify the corresponding conditional dependency (especially for system packages like gcc which provides libstdc++.so.6 and libgcc_s.so.1). Bug: https://bugs.gentoo.org/919311 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/resolver/circular_dependency.py | 15 +-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/_emerge/resolver/circular_dependency.py b/lib/_emerge/resolver/circular_dependency.py index c88b18a574..cd612e3130 100644 --- a/lib/_emerge/resolver/circular_dependency.py +++ b/lib/_emerge/resolver/circular_dependency.py @@ -1,4 +1,4 @@ -# Copyright 2010-2020 Gentoo Authors +# Copyright 2010-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import logging @@ -132,9 +132,20 @@ class circular_dependency_handler: for ppkg, atom in parent_atoms: if ppkg == parent: changed_parent = ppkg -parent_atom = atom.unevaluated_atom +parent_atom = atom break +if parent_atom.package: +parent_atom = parent_atom.unevaluated_atom +else: +# Treat soname deps as unconditional for now. In some +# cases they can be avoided by a rebuild with changed +# USE, but ebuilds sometimes do not specify the +# corresponding conditional dependency (especially for +# system packages like gcc which provides libstdc++.so.6 +# and libgcc_s.so.1). +continue + try: affecting_use = extract_affecting_use( dep, parent_atom, eapi=parent.eapi
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 09f642b894eff3a607026b1edf417f5bceee95f1
Author: Eli Schwartz gmail com>
AuthorDate: Sun Nov 26 02:25:06 2023 +
Commit: Sam James gentoo org>
CommitDate: Mon Dec 4 06:45:45 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=09f642b8
emerge: avoid spamming too much info about unused binpkgs, redux
Specifically:
- many packages can be participating in the recursive deptree, but not
get selected for actual merging, and may not even be installed at all;
these were still reported as changed-deps although the packages
themselves were not shown in show_merge_list
Previously done for mismatched-USE in commit:
bb82666b48e18f448661a1a8bf6a39b773cc4b1c
Before:
```
Local copy of remote index is up-to-date and will be used.
Local copy of remote index is up-to-date and will be used.
These are the packages that would be merged, in order:
Calculating dependencies... done!
Dependency resolution took 17.62 s.
[binary U ] dev-python/ensurepip-pip-23.3.1-1 [23.2.1]
[binary U ] sys-devel/binutils-2.40-r9-1 [2.40-r5] USE="-debuginfod%"
[binary U ] dev-python/trove-classifiers-2023.11.14-1 [2023.10.18]
[ebuild U ] dev-python/traitlets-5.13.0 [5.11.2] PYTHON_TARGETS="(-pypy3)"
[ebuild U ] dev-python/pip-23.3.1 [23.2.1]
[ebuild U ] dev-python/pycurl-7.45.2-r1 [7.45.2]
[ebuild U ] dev-python/cachetools-5.3.2 [5.3.1]
[ebuild U ] dev-python/pytest-7.4.3 [7.4.2]
[ebuild U ] dev-python/executing-2.0.1 [1.2.0] PYTHON_TARGETS="(-pypy3)
-python3_12%"
[ebuild U ] dev-python/virtualenv-20.24.7 [20.24.5]
!!! The following binary packages have been ignored due to changed dependencies:
kde-plasma/kde-cli-tools-5.27.9::gentoo
kde-plasma/kde-cli-tools-5.27.8::gentoo
NOTE: The --binpkg-changed-deps=n option will prevent emerge
from ignoring these binary packages if possible.
Using --binpkg-changed-deps=y will silence this warning.
```
After:
```
Local copy of remote index is up-to-date and will be used.
Local copy of remote index is up-to-date and will be used.
These are the packages that would be merged, in order:
Calculating dependencies... done!
Dependency resolution took 17.62 s.
[binary U ] dev-python/ensurepip-pip-23.3.1-1 [23.2.1]
[binary U ] sys-devel/binutils-2.40-r9-1 [2.40-r5] USE="-debuginfod%"
[binary U ] dev-python/trove-classifiers-2023.11.14-1 [2023.10.18]
[ebuild U ] dev-python/traitlets-5.13.0 [5.11.2] PYTHON_TARGETS="(-pypy3)"
[ebuild U ] dev-python/pip-23.3.1 [23.2.1]
[ebuild U ] dev-python/pycurl-7.45.2-r1 [7.45.2]
[ebuild U ] dev-python/cachetools-5.3.2 [5.3.1]
[ebuild U ] dev-python/pytest-7.4.3 [7.4.2]
[ebuild U ] dev-python/executing-2.0.1 [1.2.0] PYTHON_TARGETS="(-pypy3)
-python3_12%"
[ebuild U ] dev-python/virtualenv-20.24.7 [20.24.5]
```
Signed-off-by: Eli Schwartz gmail.com>
Closes: https://github.com/gentoo/portage/pull/1194
Signed-off-by: Sam James gentoo.org>
lib/_emerge/depgraph.py | 28 ++--
1 file changed, 22 insertions(+), 6 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index d3b5756429..9b09701021 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1288,17 +1288,33 @@ class depgraph:
writemsg(line + "\n", noiselevel=-1)
def _show_ignored_binaries_changed_deps(self, changed_deps):
-writemsg(
-"\n!!! The following binary packages have been "
-"ignored due to changed dependencies:\n\n",
-noiselevel=-1,
-)
+merging = {
+(pkg.root, pkg.cpv)
+for pkg in self._dynamic_config._displayed_list or ()
+if isinstance(pkg, Package)
+}
+messages = []
for pkg in changed_deps:
+# Don't include recursive deps which aren't in the merge list
anyway.
+if (pkg.root, pkg.cpv) not in merging:
+continue
+
msg = f" {pkg.cpv}{_repo_separator}{pkg.repo}"
if pkg.root_config.settings["ROOT"] != "/":
msg += f" for {pkg.root}"
-writemsg(f"{msg}\n", noiselevel=-1)
+messages.append(f"{msg}\n")
+
+if not messages:
+return
+
+writemsg(
+"\n!!! The following binary packages have been "
+"ignored due to changed dependencies:\n\n",
+noiselevel=-1,
+)
+for line in messages:
+writemsg(line, noiselevel=-1)
msg = [
"",
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 5977d8cba5bae12248188960b189012a318c8fc1
Author: Zac Medico gentoo org>
AuthorDate: Wed Nov 29 20:03:02 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Nov 29 20:04:10 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5977d8cb
depgraph: Handle empty leaves during runtime cycle topological sort
Fixes: 31832c7faf5b ("Optimize runtime cycle ignore_priority leaf selection
loop for topological sort")
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 15c3e3ca7b..d3b5756429 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -9485,10 +9485,10 @@ class depgraph:
)
if leaves:
# Select leaves with minimum ignore_priority,
-# in order to ingore as few deps as possible.
+# in order to ignore as few deps as possible.
break
-if leaves is None:
+if not leaves:
leaves = [cycle_digraph.order[-1]]
# Prefer installed leaves, in order to avoid
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: ec938c4599e3250d6fd38b9f381d57376bd66df6 Author: Zac Medico gentoo org> AuthorDate: Wed Nov 29 15:57:09 2023 + Commit: Zac Medico gentoo org> CommitDate: Wed Nov 29 19:56:19 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ec938c45 Account for runtime_slot_op in DepPriority sort for digraph.debug_print() Since 2e298ea7ba36, the digraph.debug_print() output may not display runtime_slot_op when appropriate, since it relies on DepPriority sort order to determine which priority is most relevant to display. Adjust DepPriority sort order to account for this, which makes the debug output for AlternativesGzipTestCase display runtime_slot_op where appropriate: runtime cycle digraph (9 nodes): (app-alternatives/gzip-1:0/0::test_repo, ebuild scheduled for merge) depends on (app-arch/pigz-2.8:0/0::test_repo, ebuild scheduled for merge) (runtime) (sys-libs/zlib-1.3-r1:0/1::test_repo, ebuild scheduled for merge) depends on (sys-devel/automake-1.16.5-r1:0/0::test_repo, installed) (buildtime) (sys-devel/automake-1.16.5-r1:0/0::test_repo, installed) depends on (dev-lang/perl-5.36.1-r3:0/0::test_repo, installed) (runtime) (app-alternatives/gzip-1:0/0::test_repo, ebuild scheduled for merge) (optional) (app-arch/pigz-2.8:0/0::test_repo, ebuild scheduled for merge) depends on (sys-libs/zlib-1.3-r1:0/1::test_repo, ebuild scheduled for merge) (buildtime) (app-alternatives/gzip-1:0/0::test_repo, ebuild scheduled for merge) (runtime_post) (dev-lang/perl-5.36.1-r3:0/0::test_repo, installed) depends on (sys-libs/zlib-1.3-r1:0/1::test_repo, ebuild scheduled for merge) (runtime) (virtual/libcrypt-2-r1:0/2::test_repo, installed) (runtime_slot_op) (sys-libs/glibc-2.37-r7:0/0::test_repo, installed) depends on (dev-lang/perl-5.36.1-r3:0/0::test_repo, installed) (runtime) (sys-apps/locale-gen-2.23-r1:0/0::test_repo, installed) (runtime) (sys-apps/locale-gen-2.23-r1:0/0::test_repo, installed) depends on (app-alternatives/gzip-1:0/0::test_repo, ebuild scheduled for merge) (runtime) (virtual/libcrypt-2-r1:0/2::test_repo, installed) depends on (sys-libs/libxcrypt-4.4.36:0/0::test_repo, installed) (runtime) (sys-libs/libxcrypt-4.4.36:0/0::test_repo, installed) depends on (sys-libs/glibc-2.37-r7:0/0::test_repo, installed) (runtime) Bug: https://bugs.gentoo.org/918683 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/DepPriority.py | 19 +++ 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/_emerge/DepPriority.py b/lib/_emerge/DepPriority.py index ed4adc5ab7..99d38477e2 100644 --- a/lib/_emerge/DepPriority.py +++ b/lib/_emerge/DepPriority.py @@ -19,24 +19,27 @@ class DepPriority(AbstractDepPriority): buildtime_slot_op 0 buildtime -1 -runtime-2 -runtime_post -3 -optional -4 -(none of the above)-5 +runtime_slot_op-2 +runtime-3 +runtime_post -4 +optional -5 +(none of the above)-6 """ if self.optional: -return -4 +return -5 if self.buildtime_slot_op: return 0 if self.buildtime: return -1 -if self.runtime: +if self.runtime_slot_op: return -2 -if self.runtime_post: +if self.runtime: return -3 -return -5 +if self.runtime_post: +return -4 +return -6 def __str__(self): if self.ignored:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 9823f70c6e4ef3cdd6abb4d9fc599ce02a138125 Author: Zac Medico gentoo org> AuthorDate: Tue Nov 28 23:58:23 2023 + Commit: Zac Medico gentoo org> CommitDate: Wed Nov 29 00:00:18 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9823f70c Prefer installed leaves in runtime cycle topological sort In order to avoid possibly merging a package too early, prefer installed leaves in runtime cycle topological sort. This fixes an AlternativesGzipTestCase failure that arose after 2e298ea7ba36 caused leaves to be selected in a slightly different order. Bug: https://bugs.gentoo.org/917259 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/depgraph.py | 6 ++ 1 file changed, 6 insertions(+) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index da37f980ad..9f041f83a9 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -9498,6 +9498,12 @@ class depgraph: if smallest_leaves is None: smallest_leaves = [cycle_digraph.order[-1]] +# Prefer installed leaves, in order to avoid +# merging something too early. +installed_leaves = [pkg for pkg in smallest_leaves if pkg.installed] +if installed_leaves: +smallest_leaves = installed_leaves + # Only harvest one node at a time, in order to # minimize the number of ignored dependencies. cycle_digraph.remove(smallest_leaves[0])
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 9b711229ef913e8772e9a3a1cce744c87ee468cf Author: Zac Medico gentoo org> AuthorDate: Mon Nov 27 01:55:04 2023 + Commit: Zac Medico gentoo org> CommitDate: Tue Nov 28 22:47:53 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9b711229 Cancel prefetchers that have not started yet Cancel prefetchers that have not started yet. This will cause the parallel-fetch SequentialTaskQueue to simply discard them, and a regular fetcher will be instantiated immediately. This prevents errors when accessing the pkg_path attribute of BinpkgPrefetcher instances that have not started yet. Bug: https://bugs.gentoo.org/918636 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/Binpkg.py| 6 +- lib/_emerge/CompositeTask.py | 8 +++- lib/_emerge/EbuildBuild.py | 4 lib/_emerge/Scheduler.py | 8 +++- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py index fac8722064..9b1036538a 100644 --- a/lib/_emerge/Binpkg.py +++ b/lib/_emerge/Binpkg.py @@ -1,4 +1,4 @@ -# Copyright 1999-2021 Gentoo Authors +# Copyright 1999-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import functools @@ -110,6 +110,10 @@ class Binpkg(CompositeTask): # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher +if prefetcher is not None and not prefetcher.isAlive(): +# Cancel it because it hasn't started yet. +prefetcher.cancel() +self.prefetcher = prefetcher = None if prefetcher is None: pass elif prefetcher.isAlive() and prefetcher.poll() is None: diff --git a/lib/_emerge/CompositeTask.py b/lib/_emerge/CompositeTask.py index a0203e71f3..50c1b8c5f3 100644 --- a/lib/_emerge/CompositeTask.py +++ b/lib/_emerge/CompositeTask.py @@ -1,4 +1,4 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from _emerge.AsynchronousTask import AsynchronousTask @@ -10,6 +10,12 @@ class CompositeTask(AsynchronousTask): _TASK_QUEUED = -1 +def isAlive(self): +""" +Returns True if started and returncode has not been set. +""" +return self.returncode is None and self._current_task is not None + def _cancel(self): if self._current_task is not None: if self._current_task is self._TASK_QUEUED: diff --git a/lib/_emerge/EbuildBuild.py b/lib/_emerge/EbuildBuild.py index 079a67ffbe..81cbfdc085 100644 --- a/lib/_emerge/EbuildBuild.py +++ b/lib/_emerge/EbuildBuild.py @@ -96,6 +96,10 @@ class EbuildBuild(CompositeTask): return prefetcher = self.prefetcher +if prefetcher is not None and not prefetcher.isAlive(): +# Cancel it because it hasn't started yet. +prefetcher.cancel() +self.prefetcher = prefetcher = None if prefetcher is None: pass elif prefetcher.isAlive() and prefetcher.poll() is None: diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index e1179e0f13..66eca4c650 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -940,6 +940,10 @@ class Scheduler(PollScheduler): # is consuming time here. if bintree.isremote(x.cpv): fetcher = self._get_prefetcher(x) +if fetcher is not None and not fetcher.isAlive(): +# Cancel it because it hasn't started yet. +fetcher.cancel() +fetcher = None if fetcher is None: fetcher = BinpkgFetcher(pkg=x, scheduler=loop) fetcher.start() @@ -1963,7 +1967,9 @@ class Scheduler(PollScheduler): # CPython 2.7, so it may be possible for CPython to raise KeyError # here as well. prefetcher = None -if prefetcher is not None and not prefetcher.isAlive(): +if prefetcher is not None and ( +prefetcher.cancelled or not prefetcher.isAlive() +): try: self._task_queues.fetch._task_queue.remove(prefetcher) except ValueError:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 3487594cd8f46a5c83caaab3a9425321443e5efc
Author: Zac Medico gentoo org>
AuthorDate: Tue Nov 28 04:58:07 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Nov 28 22:41:45 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3487594c
Increase ignore_priority during topological sort for runtime cycle
Fix AlternativesGzipTestCase by increasing ignore_priority in
order to find smaller groups of leaf nodes during topological
sort for runtime cycles. This causes some changes in merge
order for MergeOrderTestCase, but they appear to be acceptable
since they prevent temporarily unsatisfied RDEPEND by relying
on satisfied installed build-time dependencies.
Bug: https://bugs.gentoo.org/917259
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py| 27 --
.../tests/resolver/test_alternatives_gzip.py | 8 +++
lib/portage/tests/resolver/test_merge_order.py | 20
.../tests/resolver/test_rebuild_ghostscript.py | 2 +-
4 files changed, 35 insertions(+), 22 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 29eadba3d1..da37f980ad 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -9478,17 +9478,30 @@ class depgraph:
)
selected_nodes = []
while cycle_digraph:
+# Increase ignore_priority in order to find
+# smaller groups of leaf nodes. This solves
+# bug 917259 which happened because too many
+# leaves were selected at once.
+smallest_leaves = None
for ignore_priority in ignore_priorities:
leaves = cycle_digraph.leaf_nodes(
ignore_priority=ignore_priority
)
-if leaves:
-cycle_digraph.difference_update(leaves)
-selected_nodes.extend(leaves)
-break
-else:
-selected_nodes.extend(cycle_digraph)
-break
+if leaves and (
+smallest_leaves is None
+or len(leaves) < len(smallest_leaves)
+):
+smallest_leaves = leaves
+if len(smallest_leaves) == 1:
+break
+
+if smallest_leaves is None:
+smallest_leaves = [cycle_digraph.order[-1]]
+
+# Only harvest one node at a time, in order to
+# minimize the number of ignored dependencies.
+cycle_digraph.remove(smallest_leaves[0])
+selected_nodes.append(smallest_leaves[0])
if not selected_nodes and myblocker_uninstalls:
# An Uninstall task needs to be executed in order to
diff --git a/lib/portage/tests/resolver/test_alternatives_gzip.py
b/lib/portage/tests/resolver/test_alternatives_gzip.py
index 602ed1756f..e763e84640 100644
--- a/lib/portage/tests/resolver/test_alternatives_gzip.py
+++ b/lib/portage/tests/resolver/test_alternatives_gzip.py
@@ -1,8 +1,6 @@
# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import pytest
-
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
@@ -10,7 +8,6 @@ from portage.tests.resolver.ResolverPlayground import (
)
-@pytest.mark.xfail()
class AlternativesGzipTestCase(TestCase):
def testAlternativesGzip(self):
"""
@@ -19,8 +16,9 @@ class AlternativesGzipTestCase(TestCase):
find_smallest_cycle selects a large cycle and the topological
sort produces poor results when leaf_nodes returns
app-alternatives/gzip as part of a large group of nodes.
-This problem might be solved by implementing a finer-grained
-ignore_priority for leaf_nodes calls.
+This problem was solved by changing the topological sort to
+increase ignore_priority in order to select a smaller number
+of leaf nodes at a time.
"""
ebuilds = {
"app-alternatives/gzip-1": {
diff --git a/lib/portage/tests/resolver/test_merge_order.py
b/lib/portage/tests/resolver/test_merge_order.py
index 940eb3bbbe..e6d45c847b 100644
--- a/lib/portage/tests/resolver/test_merge_order.py
+++ b/lib/portage/tests/resolver/test_merge_order.py
@@ -1,4 +1,4 @@
-# Copyright 2011-2020 Gentoo Authors
+# Copyright 2011-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -382,10 +382,12 @@ class MergeOrderTestCase(TestCase):
ambiguous_merge_order=True,
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 2e298ea7ba36801a1cfba6e4cbfc16a7c05ee73d
Author: Sam James gentoo org>
AuthorDate: Tue Nov 28 06:22:44 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue Nov 28 22:07:46 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2e298ea7
DepPriority{Normal,Satisfied}Range: strengthen _ignore_runtime for runtime slot
operators
In the reported bug, net-misc/curl gets merged (binary), then dev-util/cmake
gets
bulit (from source) which fails because one of the built curl's dependencies
(net-libs/nghttp2) is missing:
```
[binary R] net-misc/curl-8.4.0::test_repo USE="http2%*" 0 KiB
[ebuild U ] dev-util/cmake-3.27.8::test_repo [3.26.5-r2::test_repo] 0 KiB
[ebuild N ] net-libs/nghttp2-1.57.0::test_repo 0 KiB
```
Zac had the idea [0] of strengthening _ignore_runtime to consider runtime slot
deps
as well, so we now get:
```
[ebuild U ] dev-util/cmake-3.27.8::test_repo [3.26.5-r2::test_repo] 0 KiB
[ebuild N ] net-libs/nghttp2-1.57.0::test_repo 0 KiB
[binary R] net-misc/curl-8.4.0::test_repo USE="http2%*" 0 KiB
```
For DepPrioritySatisfiedRange, we now allow ignoring the dep if:
* it's either a satisfied runtime slot dep, or it's not a runtime slot dep at
all, and
* the dep is satisfied or it's optional/not a build time dep.
(i.e. we now prevent ignoring the slot dep unless it's satisfied.)
For DepPriorityNormalRange, we now allow ignoring the dep if:
* it's not a runtime slot dep, and
* it's optional, or
* it's not a buildtime dep.
(i.e. we now prevent ignoring the slot dep.)
We then realise we can't ignore curl's dep on nghttp2 and come up with a better
order.
[0] https://github.com/gentoo/portage/pull/1193#issuecomment-1829178126
Bug: https://bugs.gentoo.org/918683
Thanks-to: Zac Medico gentoo.org>
Signed-off-by: Sam James gentoo.org>
lib/_emerge/DepPriorityNormalRange.py | 8 +++-
lib/_emerge/DepPrioritySatisfiedRange.py| 13 +++--
.../tests/resolver/test_runtime_cycle_merge_order.py| 9 -
3 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/lib/_emerge/DepPriorityNormalRange.py
b/lib/_emerge/DepPriorityNormalRange.py
index a85e1b9c14..d7e4381b47 100644
--- a/lib/_emerge/DepPriorityNormalRange.py
+++ b/lib/_emerge/DepPriorityNormalRange.py
@@ -37,7 +37,13 @@ class DepPriorityNormalRange:
def _ignore_runtime(cls, priority):
if priority.__class__ is not DepPriority:
return False
-return bool(priority.optional or not priority.buildtime)
+# If we ever allow "optional" runtime_slot_op, we'll need
+# to adjust this appropriately. But only build time dependencies
+# are optional right now, so it's not an issue as-is.
+return bool(
+not priority.runtime_slot_op
+and (priority.optional or not priority.buildtime)
+)
ignore_medium = _ignore_runtime
ignore_medium_soft = _ignore_runtime_post
diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py
b/lib/_emerge/DepPrioritySatisfiedRange.py
index 0633a5e1c2..0d42e7613d 100644
--- a/lib/_emerge/DepPrioritySatisfiedRange.py
+++ b/lib/_emerge/DepPrioritySatisfiedRange.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2013 Gentoo Foundation
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.DepPriority import DepPriority
@@ -89,7 +89,16 @@ class DepPrioritySatisfiedRange:
def _ignore_runtime(cls, priority):
if priority.__class__ is not DepPriority:
return False
-return bool(priority.satisfied or priority.optional or not
priority.buildtime)
+# We could split this up into 2 variants (ignoring satisfied
+# runtime_slot_op, and not) if we need more granularity for
ignore_priority
+# in future.
+return bool(
+(
+(not priority.runtime_slot_op)
+or (priority.satisfied and priority.runtime_slot_op)
+)
+and (priority.satisfied or priority.optional or not
priority.buildtime)
+)
ignore_medium = _ignore_runtime
ignore_medium_soft = _ignore_satisfied_buildtime_slot_op
diff --git a/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
b/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
index 26850ccad2..ed329aa097 100644
--- a/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
+++ b/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
@@ -1,4 +1,4 @@
-# Copyright 2016 Gentoo Foundation
+# Copyright 2016-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -7,8 +7,6 @@ from portage.tests.resolver.ResolverPlayground import (
ResolverPlaygroundTestCase,
)
-import pytest
-
class RuntimeCycleMergeOrderTestCase(TestCase):
def testRuntimeCycleMergeOrder(self):
@@ -77,7
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 49e01d041c74680a81860b819daff812d83df02f Author: Zac Medico gentoo org> AuthorDate: Tue Nov 28 03:42:17 2023 + Commit: Zac Medico gentoo org> CommitDate: Tue Nov 28 03:42:17 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=49e01d04 find_smallest_cycle: Optimize to traverse fewer nodes If gather_deps is traversing a cycle that is greater than or equal to the size of the current smallest_cycle, then abort early. Also abort early if we traverse a node encountered in a previous cycle for the same ignore_priority, since that means the two cycles are identical. On my laptop, this brings the emerge -pe @world time down to 3m28.884s, compared to 10m44.268s with portage-3.0.55. Bug: https://bugs.gentoo.org/918682 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/depgraph.py| 36 -- .../tests/resolver/test_rebuild_ghostscript.py | 2 +- .../resolver/test_runtime_cycle_merge_order.py | 7 +++-- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index e4305c18c9..29eadba3d1 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -9151,7 +9151,14 @@ class depgraph: asap_nodes.extend(libc_pkgs) -def gather_deps(ignore_priority, mergeable_nodes, selected_nodes, node): +def gather_deps( +ignore_priority, +mergeable_nodes, +selected_nodes, +node, +smallest_cycle=None, +traversed_nodes=None, +): """ Recursively gather a group of nodes that RDEPEND on eachother. This ensures that they are merged as a group @@ -9171,10 +9178,24 @@ class depgraph: # RDEPENDs installed first, but ignore uninstalls # (these occur when new portage blocks an older package version). return False +if traversed_nodes is not None: +if node in traversed_nodes: +# Identical to a previously traversed cycle. +return False +traversed_nodes.add(node) selected_nodes.add(node) +if smallest_cycle is not None and len(selected_nodes) >= len( +smallest_cycle +): +return False for child in mygraph.child_nodes(node, ignore_priority=ignore_priority): if not gather_deps( -ignore_priority, mergeable_nodes, selected_nodes, child +ignore_priority, +mergeable_nodes, +selected_nodes, +child, +smallest_cycle=smallest_cycle, +traversed_nodes=traversed_nodes, ): return False return True @@ -9332,12 +9353,21 @@ class depgraph: local_priority_range.MEDIUM_SOFT + 1, ) ): +# Traversed nodes for current priority +traversed_nodes = set() for node in nodes: if not mygraph.parent_nodes(node): continue +if node in traversed_nodes: +continue selected_nodes = set() if gather_deps( -priority, mergeable_nodes, selected_nodes, node +priority, +mergeable_nodes, +selected_nodes, +node, +smallest_cycle=smallest_cycle, +traversed_nodes=traversed_nodes, ): if smallest_cycle is None or len(selected_nodes) < len( smallest_cycle diff --git a/lib/portage/tests/resolver/test_rebuild_ghostscript.py b/lib/portage/tests/resolver/test_rebuild_ghostscript.py index 8d7cbb1f92..88dc2b5fc3 100644 --- a/lib/portage/tests/resolver/test_rebuild_ghostscript.py +++ b/lib/portage/tests/resolver/test_rebuild_ghostscript.py @@ -137,9 +137,9 @@ class RebuildGhostscriptTestCase(TestCase): mergelist=[ "sys-apps/dbus-1.15.6", "x11-libs/gtk+-3.24.38", +"app-text/ghostscript-gpl-10.01.2", "net-print/cups-2.4.6", "net-dns/avahi-0.8-r7", -"app-text/ghostscript-gpl-10.01.2", "app-text/libspectre-0.2.12", "x11-libs/goffice-0.10.55", ], diff --git a/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: c8e599af298a500334e6566d1518b35a531ec99e
Author: Zac Medico gentoo org>
AuthorDate: Sun Nov 19 19:52:01 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Nov 25 06:31:31 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c8e599af
_run_pkg_pretend: Refer to emerge-fetch.log for binpkg parallel-fetch
If there is a parallel-fetch task fetching a binpkg in the
background, refer to emerge-fetch.log for status updates,
just like the Binpkg class does prior to pkg_setup.
Bug: https://bugs.gentoo.org/760893
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/Scheduler.py | 10 ++
1 file changed, 10 insertions(+)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index ceff2be0f7..e1179e0f13 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -948,6 +948,16 @@ class Scheduler(PollScheduler):
# handles fetch, verification, and the
# bintree.inject call which moves the file.
fetched = fetcher.pkg_path
+else:
+msg = (
+"Fetching in the background:",
+fetcher.pkg_path,
+"To view fetch progress, run in another
terminal:",
+f"tail -f {self._fetch_log}",
+)
+out = portage.output.EOutput()
+for l in msg:
+out.einfo(l)
if await fetcher.async_wait() != os.EX_OK:
failures += 1
self._record_pkg_failure(x, settings,
fetcher.returncode)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 9206d5a75ecd2d9ae0fe63e57d28aa8061b5927e
Author: Zac Medico gentoo org>
AuthorDate: Sat Nov 18 17:07:59 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Nov 19 04:15:22 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9206d5a7
find_smallest_cycle: Increase ignore_priority to find smaller cycles
Fix AlternativesGzipTestCase by increasing ignore_priority in
order to find smaller cycles. This causes some changes in merge
order for MergeOrderTestCase, but they appear to be acceptable
since they prevent temporarily unsatisfied RDEPEND by relying
on satisfied installed build-time dependencies.
Add a test case for bug 690436, since the change to merge order
in MergeOrderTestCase is related (see commit 680276cc4d4f).
Bug: https://bugs.gentoo.org/690436
Bug: https://bugs.gentoo.org/917259
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py| 7 +-
lib/portage/tests/resolver/meson.build | 1 +
.../tests/resolver/test_alternatives_gzip.py | 7 +-
lib/portage/tests/resolver/test_merge_order.py | 24 ++--
.../tests/resolver/test_rebuild_ghostscript.py | 160 +
5 files changed, 180 insertions(+), 19 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index e4305c18c9..0d3b37c698 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -9345,9 +9345,10 @@ class depgraph:
smallest_cycle = selected_nodes
ignore_priority = priority
-# Exit this loop with the lowest possible priority,
which
-# minimizes the use of installed packages to break
cycles.
-if smallest_cycle is not None:
+if smallest_cycle is not None and len(smallest_cycle)
== 1:
+# The cycle can't get any smaller than this,
+# so there is no need to search further since
+# we try to minimize ignore_priority.
break
return smallest_cycle, ignore_priority
diff --git a/lib/portage/tests/resolver/meson.build
b/lib/portage/tests/resolver/meson.build
index 7d2bd367d4..770027ac47 100644
--- a/lib/portage/tests/resolver/meson.build
+++ b/lib/portage/tests/resolver/meson.build
@@ -49,6 +49,7 @@ py.install_sources(
'test_profile_default_eapi.py',
'test_profile_package_set.py',
'test_rebuild.py',
+'test_rebuild_ghostscript.py',
'test_regular_slot_change_without_revbump.py',
'test_required_use.py',
'test_runtime_cycle_merge_order.py',
diff --git a/lib/portage/tests/resolver/test_alternatives_gzip.py
b/lib/portage/tests/resolver/test_alternatives_gzip.py
index 602ed1756f..7cd1da25f1 100644
--- a/lib/portage/tests/resolver/test_alternatives_gzip.py
+++ b/lib/portage/tests/resolver/test_alternatives_gzip.py
@@ -1,8 +1,6 @@
# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import pytest
-
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
@@ -10,7 +8,6 @@ from portage.tests.resolver.ResolverPlayground import (
)
-@pytest.mark.xfail()
class AlternativesGzipTestCase(TestCase):
def testAlternativesGzip(self):
"""
@@ -19,8 +16,8 @@ class AlternativesGzipTestCase(TestCase):
find_smallest_cycle selects a large cycle and the topological
sort produces poor results when leaf_nodes returns
app-alternatives/gzip as part of a large group of nodes.
-This problem might be solved by implementing a finer-grained
-ignore_priority for leaf_nodes calls.
+This problem was solved by increasing ignore_priority in order
+to find a smaller cycle.
"""
ebuilds = {
"app-alternatives/gzip-1": {
diff --git a/lib/portage/tests/resolver/test_merge_order.py
b/lib/portage/tests/resolver/test_merge_order.py
index 940eb3bbbe..671543ca29 100644
--- a/lib/portage/tests/resolver/test_merge_order.py
+++ b/lib/portage/tests/resolver/test_merge_order.py
@@ -382,10 +382,12 @@ class MergeOrderTestCase(TestCase):
ambiguous_merge_order=True,
# The following merge order assertion reflects optimal order
for
# a circular relationship which is DEPEND in one direction and
-# RDEPEND in the other.
-merge_order_assertions=(
-("app-misc/circ-buildtime-a-1",
"app-misc/circ-buildtime-c-1"),
-),
+# RDEPEND in the other. However, it is not respected because
+# it would result in a temporarily broken RDEPEND, so we
instead
+# rely on satisfied installed build-time dependencies.
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: bf61ab8c1ae0eb3311fea20f1bdc5cf7d69fd586
Author: Zac Medico gentoo org>
AuthorDate: Mon Nov 13 23:17:26 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Nov 13 23:17:30 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bf61ab8c
_emerge: Fix _failed_pkg cpv AttributeError
Bug: https://bugs.gentoo.org/917315
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/Scheduler.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index c7e465731a..ceff2be0f7 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from collections import deque
@@ -1260,7 +1260,8 @@ class Scheduler(PollScheduler):
and not mod_echo_output
):
failed_pkg_map = {}
-for pkg in self._failed_pkgs_all:
+for failed_pkg in self._failed_pkgs_all:
+pkg = failed_pkg.pkg
failed_pkg_map[(pkg.cpv, pkg.root)] = pkg
for mysettings, key, logentries in self._failed_pkgs_die_msgs:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 37ee6ef700f4512999dc2e3bd8fcd8a8d9d36315 Author: Sam James gentoo org> AuthorDate: Sat Nov 11 03:03:29 2023 + Commit: Sam James gentoo org> CommitDate: Sat Nov 11 07:23:08 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=37ee6ef7 _emerge: fix _eval_visiblity typo Signed-off-by: Sam James gentoo.org> lib/_emerge/Package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/_emerge/Package.py b/lib/_emerge/Package.py index 873edbf81d..79011380d5 100644 --- a/lib/_emerge/Package.py +++ b/lib/_emerge/Package.py @@ -222,7 +222,7 @@ class Package(Task): @property def visible(self): if self._visible is None: -self._visible = self._eval_visiblity(self.masks) +self._visible = self._eval_visibility(self.masks) return self._visible @property @@ -473,7 +473,7 @@ class Package(Task): return masks -def _eval_visiblity(self, masks): +def _eval_visibility(self, masks): if masks is not False: if "EAPI.unsupported" in masks: return False
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: cfb856419711784fce2c5d54781338729cb94287
Author: Zac Medico gentoo org>
AuthorDate: Fri Nov 10 05:34:53 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Nov 10 05:47:15 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cfb85641
emerge: Use finally instead of atexit for xtermTitleReset
The xtermTitleReset function will trigger a RuntimeError
when called via an atexit hook in python 3.12, since it
forks to run PROMPT_COMMAND. Avoid this problem by calling
it in a finally block instead of atexit.
Bug: https://bugs.gentoo.org/917033
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/actions.py | 10 --
lib/_emerge/main.py| 10 ++
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 07d477a046..37264114e0 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -54,7 +54,6 @@ from portage.output import (
darkgreen,
red,
xtermTitle,
-xtermTitleReset,
)
good = create_color_func("GOOD")
@@ -3828,15 +3827,6 @@ def run_action(emerge_config):
signal.signal(signal.SIGTERM, emergeexitsig)
-def emergeexit():
-"""This gets out final log message in before we quit."""
-if "--pretend" not in emerge_config.opts:
-emergelog(xterm_titles, " *** terminating.")
-if xterm_titles:
-xtermTitleReset()
-
-portage.atexit_register(emergeexit)
-
if emerge_config.action in ("config", "metadata", "regen", "sync"):
if "--pretend" in emerge_config.opts:
sys.stderr.write(
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index d92f069901..6a52404c88 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -12,9 +12,11 @@ portage.proxy.lazyimport.lazyimport(
globals(),
"logging",
"portage.dep:Atom",
+"portage.output:xtermTitleReset",
"portage.util:writemsg_level",
"textwrap",
"_emerge.actions:load_emerge_config,run_action," +
"validate_ebuild_environment",
+"_emerge.emergelog:emergelog",
"_emerge.help:emerge_help",
"_emerge.is_valid_package_atom:insert_category_into_atom",
)
@@ -1310,3 +1312,11 @@ def emerge_main(args: Optional[list[str]] = None):
if "porttree" in x.lazy_items:
continue
x["porttree"].dbapi.close_caches()
+
+# This gets out final log message in before we quit.
+xterm_titles = "notitles" not in
emerge_config.target_config.settings.features
+if "--pretend" not in emerge_config.opts:
+emergelog(xterm_titles, " *** terminating.")
+
+if xterm_titles:
+xtermTitleReset()
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/, /
commit: c56ca69564452198039e17b3a07f24f5b87a6852
Author: Sam James gentoo org>
AuthorDate: Mon Nov 6 15:03:10 2023 +
Commit: Sam James gentoo org>
CommitDate: Mon Nov 6 15:57:56 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c56ca695
emerge: fix _show_ignored_binaries_respect_use with incomplete depgraph
I've gone for the simpler solution of just using an empty tuple where the
merge list is empty to preserve prior behaviour with what we do (or do not)
display wrt skipped binaries.
Bug: https://bugs.gentoo.org/916614
Signed-off-by: Sam James gentoo.org>
NEWS| 2 +
lib/_emerge/depgraph.py | 4 +-
lib/portage/tests/resolver/test_useflags.py | 89 +
3 files changed, 94 insertions(+), 1 deletion(-)
diff --git a/NEWS b/NEWS
index 4a33559a33..89d9335275 100644
--- a/NEWS
+++ b/NEWS
@@ -15,6 +15,8 @@ Bug fixes:
* Avoid crash with incomplete depgraph for binpkg-respect-use notice (bug
#916614).
+* Avoid crash with blockers in depgraph for binpkg-respect-use notice (bug
#916336).
+
portage-3.0.54 (2023-10-25)
--
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index d1fed0d652..e4305c18c9 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1234,11 +1234,13 @@ class depgraph:
def _show_ignored_binaries_respect_use(self, respect_use):
seen = {}
messages = []
+
merging = {
(pkg.root, pkg.cpv)
-for pkg in self._dynamic_config._displayed_list
+for pkg in self._dynamic_config._displayed_list or ()
if isinstance(pkg, Package)
}
+
for pkg, flags in respect_use.items():
# Don't include recursive deps which aren't in the merge list
anyway.
if (pkg.root, pkg.cpv) not in merging:
diff --git a/lib/portage/tests/resolver/test_useflags.py
b/lib/portage/tests/resolver/test_useflags.py
index ee9f4f2763..86684f7f28 100644
--- a/lib/portage/tests/resolver/test_useflags.py
+++ b/lib/portage/tests/resolver/test_useflags.py
@@ -229,3 +229,92 @@ class UseFlagsTestCase(TestCase):
)
finally:
playground.cleanup()
+
+def testNoMergeBinpkgRespectUse(self):
+"""
+Testcase for bug #916614 where an incomplete depgraph may be fed into
+_show_ignored_binaries_respect_use.
+
+We use a mix of +/-abi_x86_32 to trigger the binpkg-respect-use notice
+and depend on a non-existent package in one of the available ebuilds we
+queue to reinstall to trigger an aborted calculation.
+"""
+ebuilds = {
+"dev-libs/A-2": {
+"EAPI": "7",
+"IUSE": "abi_x86_32",
+},
+"dev-libs/B-1": {
+"IUSE": "abi_x86_32",
+"RDEPEND": "=dev-libs/A-1",
+},
+}
+
+installed = {
+"dev-libs/B-1": {
+"IUSE": "abi_x86_32",
+"USE": "abi_x86_32",
+},
+"dev-libs/A-1": {
+"IUSE": "abi_x86_32",
+"USE": "abi_x86_32",
+},
+}
+
+binpkgs = {
+"dev-libs/A-2": {
+"IUSE": "abi_x86_32",
+"USE": "abi_x86_32",
+},
+"dev-libs/B-1": {
+"IUSE": "abi_x86_32",
+"USE": "",
+"BUILD_ID": "2",
+"BUILD_TIME": "2",
+},
+}
+
+user_config = {
+"make.conf": (
+'FEATURES="binpkg-multi-instance"',
+'USE="abi_x86_32 abi_x86_32"',
+),
+}
+
+world = ("dev-libs/A",)
+
+test_cases = (
+ResolverPlaygroundTestCase(
+["@installed"],
+options={
+"--verbose": "y",
+"--emptytree": True,
+"--usepkg": True,
+},
+success=False,
+mergelist=["[binary]dev-libs/A-2", "dev-libs/B-1"],
+slot_collision_solutions=[],
+),
+)
+
+for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+with self.subTest(binpkg_format=binpkg_format):
+print(colorize("HILITE", binpkg_format), end=" ... ")
+sys.stdout.flush()
+user_config["make.conf"] +=
(f'BINPKG_FORMAT="{binpkg_format}"',)
+playground = ResolverPlayground(
+ebuilds=ebuilds,
+binpkgs=binpkgs,
+installed=installed,
+user_config=user_config,
+world=world,
+)
+
+try:
+for test_case in test_cases:
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/, /
commit: 262b8c9213c6698be4a7e2ce88536568899e6c2a
Author: Sam James gentoo org>
AuthorDate: Mon Oct 30 05:30:04 2023 +
Commit: Sam James gentoo org>
CommitDate: Mon Nov 6 15:57:56 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=262b8c92
emerge: fix binpkg-respect-use notice with blockers
Items in _dynamic_config._displayed_list might be blockers, so filter those out.
Bug: https://bugs.gentoo.org/916336
Fixes: bb82666b48e18f448661a1a8bf6a39b773cc4b1c
Signed-off-by: Sam James gentoo.org>
NEWS| 2 +
lib/_emerge/depgraph.py | 6 +-
lib/portage/tests/resolver/test_useflags.py | 87 +
3 files changed, 94 insertions(+), 1 deletion(-)
diff --git a/NEWS b/NEWS
index be551de33d..4a33559a33 100644
--- a/NEWS
+++ b/NEWS
@@ -13,6 +13,8 @@ Bug fixes:
* Convert portageq helper to a function to avoid breaking external callers
(bug #916287, bug #916296).
+* Avoid crash with incomplete depgraph for binpkg-respect-use notice (bug
#916614).
+
portage-3.0.54 (2023-10-25)
--
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 85845dc1e2..0717e0429f 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1234,7 +1234,11 @@ class depgraph:
def _show_ignored_binaries_respect_use(self, respect_use):
seen = {}
messages = []
-merging = {pkg.cpv for pkg in self._dynamic_config._displayed_list}
+merging = {
+pkg.cpv
+for pkg in self._dynamic_config._displayed_list
+if isinstance(pkg, Package)
+}
for pkg, flags in respect_use.items():
# Don't include recursive deps which aren't in the merge list
anyway.
if pkg.cpv not in merging:
diff --git a/lib/portage/tests/resolver/test_useflags.py
b/lib/portage/tests/resolver/test_useflags.py
index 0af1cb5585..ee9f4f2763 100644
--- a/lib/portage/tests/resolver/test_useflags.py
+++ b/lib/portage/tests/resolver/test_useflags.py
@@ -142,3 +142,90 @@ class UseFlagsTestCase(TestCase):
)
finally:
playground.cleanup()
+
+def testBlockerBinpkgRespectUse(self):
+"""
+Test for bug #916336 where we tried to check properties of a blocker
+object which isn't a Package to be merged.
+"""
+
+ebuilds = {
+"dev-libs/A-1": {
+"EAPI": "7",
+"IUSE": "abi_x86_32",
+"RDEPEND": "dev-libs/B",
+},
+"dev-libs/B-1": {
+"EAPI": "7",
+"IUSE": "abi_x86_32",
+},
+"dev-libs/A-2": {
+"EAPI": "7",
+"IUSE": "abi_x86_32",
+"RDEPEND": "!
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: a6853d5493b7bed37e60c2e3b7536b209500ba3f
Author: Sam James gentoo org>
AuthorDate: Thu Nov 2 13:59:05 2023 +
Commit: Sam James gentoo org>
CommitDate: Mon Nov 6 15:57:56 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a6853d54
emerge: account for EROOT in _show_ignored_binaries_respect_use
Zac suggested this when reviewing the fix for bug #916336.
Bug: https://bugs.gentoo.org/916336
Signed-off-by: Sam James gentoo.org>
lib/_emerge/depgraph.py | 13 +
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 0717e0429f..d1fed0d652 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1235,23 +1235,28 @@ class depgraph:
seen = {}
messages = []
merging = {
-pkg.cpv
+(pkg.root, pkg.cpv)
for pkg in self._dynamic_config._displayed_list
if isinstance(pkg, Package)
}
for pkg, flags in respect_use.items():
# Don't include recursive deps which aren't in the merge list
anyway.
-if pkg.cpv not in merging:
+if (pkg.root, pkg.cpv) not in merging:
continue
+
flag_display = []
for flag in sorted(flags):
if flag not in pkg.use.enabled:
flag = "-" + flag
flag_display.append(flag)
flag_display = " ".join(flag_display)
+
# We don't want to list the same USE flags for multiple build IDs
-if pkg.cpv not in seen or flag_display not in seen[pkg.cpv]:
-seen.setdefault(pkg.cpv, set()).add(flag_display)
+seen.setdefault(pkg.root, dict())
+if (pkg.root, pkg.cpv) not in seen or flag_display not in
seen[pkg.root][
+pkg.cpv
+]:
+seen[pkg.root].setdefault(pkg.cpv, set()).add(flag_display)
# The user can paste this line into package.use
messages.append(f"={pkg.cpv} {flag_display}")
if pkg.root_config.settings["ROOT"] != "/":
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 6963901bebf5403be5042fcbc54f4c950608204e Author: Zac Medico gentoo org> AuthorDate: Sun Oct 22 22:50:23 2023 + Commit: Zac Medico gentoo org> CommitDate: Sun Oct 22 22:51:28 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6963901b BinpkgVerifier: multiprocessing spawn compat Solves this error with the multiprocessing spawn start method: TypeError: cannot pickle 'generator' object Bug: https://bugs.gentoo.org/916141 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/BinpkgVerifier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/_emerge/BinpkgVerifier.py b/lib/_emerge/BinpkgVerifier.py index 58691e68b6..a7917453aa 100644 --- a/lib/_emerge/BinpkgVerifier.py +++ b/lib/_emerge/BinpkgVerifier.py @@ -1,4 +1,4 @@ -# Copyright 1999-2019 Gentoo Authors +# Copyright 1999-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -59,7 +59,7 @@ class BinpkgVerifier(CompositeTask): self._start_task( FileDigester( file_path=self._pkg_path, -hash_names=(k for k in digests if k != "size"), +hash_names=[k for k in digests if k != "size"], background=self.background, logfile=self.logfile, scheduler=self.scheduler,
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: bb82666b48e18f448661a1a8bf6a39b773cc4b1c Author: Eli Schwartz gmail com> AuthorDate: Sun Oct 22 16:53:23 2023 + Commit: Sam James gentoo org> CommitDate: Sun Oct 22 22:09:32 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bb82666b emerge: avoid spamming too much info about unused binpkgs Specifically: - the binhost might have multiple binpkgs for the same cpv, same USE flags, but with different build ids, in which case the same line is repeated twice - many packages can be participating in the recursive deptree, but not get selected for actual merging, and may not even be installed at all; these were still reported as mismatched-USE although the packages themselves were not shown in show_merge_list Both cases are useless noise. Prune them before printing a status report. Before: ``` Local copy of remote index is up-to-date and will be used. Local copy of remote index is up-to-date and will be used. These are the packages that would be merged, in order: Calculating dependencies... done! Dependency resolution took 17.29 s. [binary U ] sys-apps/hwdata-0.374-1 [0.373] [binary r U ] dev-libs/libsodium-1.0.19-r1-1 [1.0.18_p20220618] [binary U ] net-dns/libidn2-2.3.4-r1-1 [2.3.4] [ebuild R] dev-python/fs-2.4.16 PYTHON_TARGETS="(-python3_12%*)" [ebuild U ] sys-apps/iproute2-6.5.0 [6.4.0] [binary U ] media-video/ffmpeg-6.0-r9-2 [6.0-r6] [ebuild rR] app-editors/vim-9.0.1627 The following packages are causing rebuilds: (dev-libs/libsodium-1.0.19-r1-1:0/26::gentoo, binary scheduled for merge) causes rebuilds for: (app-editors/vim-9.0.1627:0/0::gentoo, ebuild scheduled for merge) !!! The following binary packages have been ignored due to non matching USE: =app-shells/bash-5.1_p16-r6 -plugins =sys-apps/util-linux-2.38.1-r2 -caps split-usr =sys-apps/util-linux-2.38.1-r2 -caps split-usr =sys-process/procps-3.3.17-r2 split-usr =sys-process/procps-3.3.17-r2 split-usr -systemd =app-editors/vim-9.0.1627 -python =app-editors/vim-9.0.1627 -python =dev-lang/python-3.11.5 -lto -pgo =dev-lang/python-3.11.5 -lto -pgo =dev-lang/python-3.11.5 -lto -pgo =media-fonts/noto-20230831 X =net-misc/networkmanager-1.42.6-r2 bluetooth gtk-doc ppp =net-misc/networkmanager-1.42.6-r2 bluetooth gtk-doc ppp =net-misc/networkmanager-1.42.6-r2 bluetooth gtk-doc ppp =net-misc/networkmanager-1.42.6-r2 bluetooth gtk-doc ppp =sys-devel/gdb-13.2-r1 -lzma nls -source-highlight -xxhash -zstd =x11-base/xorg-server-21.1.8-r2 -xvfb =media-libs/libsdl2-2.28.3 gles2 pulseaudio wayland =media-libs/libsdl2-2.28.3 gles2 pipewire pulseaudio wayland =sys-devel/clang-16.0.6 abi_x86_32 =sys-devel/clang-16.0.6 -xml =app-arch/zstd-1.5.5 split-usr =app-arch/zstd-1.5.5 split-usr =sys-apps/sandbox-2.38 -abi_x86_32 =dev-python/certifi-3021.3.16-r3 -python_targets_python3_12 =dev-python/certifi-3021.3.16-r3 -python_targets_python3_12 =dev-libs/libffi-3.4.4-r1 abi_x86_32 =sys-libs/ncurses-6.4_p20230401 abi_x86_32 -gpm split-usr =sys-libs/readline-8.1_p2-r1 split-usr =dev-db/sqlite-3.43.2 -icu =dev-libs/icu-73.2 abi_x86_32 =sys-libs/gpm-1.20.7-r5 abi_x86_32 split-usr =app-crypt/pinentry-1.2.1-r3 keyring qt5 wayland =dev-libs/glib-2.76.4 sysprof =sys-apps/systemd-253.11 split-usr =dev-libs/libxml2-2.11.5 abi_x86_32 =dev-libs/libxml2-2.11.5 -icu =media-libs/mesa-23.1.8 video_cards_intel video_cards_nouveau =media-libs/mesa-23.1.8 video_cards_intel video_cards_nouveau =media-libs/mesa-23.1.8 video_cards_intel video_cards_nouveau =media-libs/mesa-23.1.8 video_cards_intel video_cards_nouveau =media-libs/mesa-23.1.8 video_cards_intel video_cards_nouveau =x11-libs/libxkbcommon-1.5.0 wayland =x11-libs/libxkbcommon-1.5.0 wayland =x11-libs/libxkbcommon-1.5.0 wayland =sys-devel/llvm-16.0.6 abi_x86_32 =sys-devel/llvm-16.0.6 -xml =x11-misc/xdg-utils-1.1.3_p20210805-r1 gnome =dev-libs/libpcre2-10.42-r1 split-usr =sys-auth/polkit-122-r1 kde =net-libs/nghttp2-1.57.0 -systemd -xml =sys-libs/glibc-2.37-r7 -multilib -stack-realign -systemd =app-crypt/gpgme-1.21.0 -qt5 =sys-devel/clang-runtime-16.0.6 -abi_x86_32 =media-sound/mpg123-1.31.3-r1 pulseaudio =dev-libs/libpcre-8.45-r2 split-usr =net-wireless/wpa_supplicant-2.10-r3 qt5 =media-libs/gst-plugins-base-1.20.6 wayland =media-libs/gst-plugins-base-1.20.6 theora wayland =media-libs/gst-plugins-base-1.20.6 wayland =net-libs/libproxy-0.5.3 kde =gnome-base/gvfs-1.50.6 gnome-online-accounts keyring =gnome-base/gvfs-1.50.6 gnome-online-accounts keyring =sys-libs/libblockdev-2.28-r1 introspection =dev-libs/libdbusmenu-16.04.0-r2 gtk =dev-libs/libdbusmenu-16.04.0-r2 gtk =dev-python/css-parser-1.0.9 -python_targets_python3_12 =dev-qt/qtbase-6.5.2-r2 nls wayland
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/process/, lib/portage/util/_async/
commit: 9a56879ec26bfb39dc3551932d664fd6bec80500
Author: Zac Medico gentoo org>
AuthorDate: Sun Oct 22 16:46:22 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Oct 22 17:42:26 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9a56879e
SpawnProcess/ForkProcess: Pass stdin via fd_pipes if not background
Bug: https://bugs.gentoo.org/916116
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/SpawnProcess.py | 2 +-
lib/portage/tests/process/test_AsyncFunction.py | 33 -
lib/portage/util/_async/ForkProcess.py | 2 +-
3 files changed, 24 insertions(+), 13 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index b4eabd07ad..72fa72c613 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -62,7 +62,7 @@ class SpawnProcess(SubProcess):
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
-if fd_pipes or self.logfile:
+if fd_pipes or self.logfile or not self.background:
master_fd, slave_fd = self._pipe(fd_pipes)
can_log = self._can_log(slave_fd)
diff --git a/lib/portage/tests/process/test_AsyncFunction.py
b/lib/portage/tests/process/test_AsyncFunction.py
index e30ff770b3..d02a3395c1 100644
--- a/lib/portage/tests/process/test_AsyncFunction.py
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -16,25 +16,36 @@ from portage.util.futures.unix_events import
_set_nonblocking
class AsyncFunctionTestCase(TestCase):
@staticmethod
-def _read_from_stdin(pr, pw):
+def _read_from_stdin(pw):
if pw is not None:
os.close(pw)
-os.dup2(pr.fileno(), sys.stdin.fileno())
return "".join(sys.stdin)
async def _testAsyncFunctionStdin(self, loop):
test_string = "1\n2\n3\n"
pr, pw = multiprocessing.Pipe(duplex=False)
-reader = AsyncFunction(
-scheduler=loop,
-target=self._read_from_stdin,
-args=(
-pr,
-pw.fileno() if multiprocessing.get_start_method() == "fork"
else None,
-),
-)
-reader.start()
+stdin_backup = os.dup(portage._get_stdin().fileno())
+os.dup2(pr.fileno(), portage._get_stdin().fileno())
pr.close()
+try:
+reader = AsyncFunction(
+# Should automatically inherit stdin as fd_pipes[0]
+# when background is False, for things like
+# emerge --sync --ask (bug 916116).
+background=False,
+scheduler=loop,
+target=self._read_from_stdin,
+args=(
+pw.fileno()
+if multiprocessing.get_start_method() == "fork"
+else None,
+),
+)
+reader.start()
+finally:
+os.dup2(stdin_backup, portage._get_stdin().fileno())
+os.close(stdin_backup)
+
_set_nonblocking(pw.fileno())
with open(pw.fileno(), mode="wb", buffering=0, closefd=False) as
pipe_write:
await _writer(pipe_write, test_string.encode("utf_8"))
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index 6d216a5c43..780545be0e 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -52,7 +52,7 @@ class ForkProcess(SpawnProcess):
'fd_pipes only supported with HAVE_SEND_HANDLE or
multiprocessing start method "fork"'
)
-if self.fd_pipes or self.logfile:
+if self.fd_pipes or self.logfile or not self.background:
# Log via multiprocessing.Pipe if necessary.
connection, self._child_connection = multiprocessing.Pipe(
duplex=self._HAVE_SEND_HANDLE
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 935d1432b2ea42fc0a49b9e3368a1766a673e1a9
Author: Zac Medico gentoo org>
AuthorDate: Sat Oct 21 01:31:44 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Oct 22 04:36:43 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=935d1432
_emerge: Fix NameError 'pkg' bug 915551
Use (pkg.cpv, pkg.root) to lookup package
instances corresponding to die messages.
Bug: https://bugs.gentoo.org/915551
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/Scheduler.py | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index 2d1872feba..c7e465731a 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -1259,8 +1259,13 @@ class Scheduler(PollScheduler):
and self._failed_pkgs_die_msgs
and not mod_echo_output
):
+failed_pkg_map = {}
+for pkg in self._failed_pkgs_all:
+failed_pkg_map[(pkg.cpv, pkg.root)] = pkg
+
for mysettings, key, logentries in self._failed_pkgs_die_msgs:
-color = "PKG_BINARY_MERGE" if pkg.built else "INFORM"
+pkg = failed_pkg_map.get((key, mysettings["EROOT"]))
+color = "PKG_BINARY_MERGE" if pkg and pkg.built else "INFORM"
root_msg = ""
if mysettings["ROOT"] != "/":
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 5f5012803b2b647f6b7528f48579a8c27c8a3fd7 Author: Zac Medico gentoo org> AuthorDate: Sat Oct 14 19:54:39 2023 + Commit: Zac Medico gentoo org> CommitDate: Sat Oct 14 19:55:49 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5f501280 depgraph: Skip _eliminate_rebuilds for --emptytree Signed-off-by: Zac Medico gentoo.org> lib/_emerge/depgraph.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 4e4452dad1..deab51718b 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -3605,6 +3605,9 @@ class depgraph: careful to obey the user's wishes if they have explicitly requested for a package to be rebuilt or reinstalled for some reason. """ +if "empty" in self._dynamic_config.myparams: +return False + modified = False selective = "selective" in self._dynamic_config.myparams for root, atom in self._dynamic_config._slot_operator_replace_installed:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 8e88810e61b6b36bd36e8fb1e5ae76715a72dcd9
Author: Zac Medico gentoo org>
AuthorDate: Thu Oct 5 04:17:10 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Oct 5 04:18:31 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8e88810e
EbuildFetcherProcess: Migrate to ForkProcess target parameter
Bug: https://bugs.gentoo.org/915099
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildFetcher.py | 35 +++
1 file changed, 23 insertions(+), 12 deletions(-)
diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py
index 7e28be4938..edabe54456 100644
--- a/lib/_emerge/EbuildFetcher.py
+++ b/lib/_emerge/EbuildFetcher.py
@@ -1,7 +1,8 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
+import functools
import io
import sys
@@ -246,6 +247,14 @@ class _EbuildFetcherProcess(ForkProcess):
self._settings = settings
self.log_filter_file = settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
+self.target = functools.partial(
+self._target,
+self._settings,
+self._get_digests,
+self._get_manifest,
+self._uri_map,
+self.fetchonly,
+)
ForkProcess._start(self)
# Free settings now since it's no longer needed in
@@ -254,25 +263,27 @@ class _EbuildFetcherProcess(ForkProcess):
settings = None
self._settings = None
-def _run(self):
+@staticmethod
+def _target(settings, get_digests, get_manifest, uri_map, fetchonly):
+"""
+TODO: Make all arguments picklable for the multiprocessing spawn start
method.
+"""
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
-portage.output.havecolor = self._settings.get("NOCOLOR") not in
("yes", "true")
+portage.output.havecolor = settings.get("NOCOLOR") not in ("yes",
"true")
# For userfetch, drop privileges for the entire fetch call, in
# order to handle DISTDIR on NFS with root_squash for bug 601252.
-if _want_userfetch(self._settings):
-_drop_privs_userfetch(self._settings)
+if _want_userfetch(settings):
+_drop_privs_userfetch(settings)
rval = 1
-allow_missing = (
-self._get_manifest().allow_missing or "digest" in
self._settings.features
-)
+allow_missing = get_manifest().allow_missing or "digest" in
settings.features
if fetch(
-self._uri_map,
-self._settings,
-fetchonly=self.fetchonly,
-digests=copy.deepcopy(self._get_digests()),
+uri_map,
+settings,
+fetchonly=fetchonly,
+digests=copy.deepcopy(get_digests()),
allow_missing_digests=allow_missing,
):
rval = os.EX_OK
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/ebuild/
commit: 919567592eb2292def12877420f280e082cc881a
Author: Zac Medico gentoo org>
AuthorDate: Tue Oct 3 19:07:37 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Oct 3 19:43:08 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=91956759
SpawnProcess: Make fd_pipes optional
Since fd_pipes is incompatible with the multiprocessing spawn start
method, allow empty fd_pipes so that ForkProcess can inherit
SpawnProcess and still be compatible with multiprocessing spawn.
Bug: https://bugs.gentoo.org/915128
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/SpawnProcess.py | 111 +++-
lib/portage/tests/ebuild/test_ipc_daemon.py | 9 ++-
2 files changed, 68 insertions(+), 52 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 8f21f05c04..0e27f29c43 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2008-2021 Gentoo Authors
+# Copyright 2008-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -62,55 +62,60 @@ class SpawnProcess(SubProcess):
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
-master_fd, slave_fd = self._pipe(fd_pipes)
-
-can_log = self._can_log(slave_fd)
-if can_log:
-log_file_path = self.logfile
-else:
-log_file_path = None
-
-null_input = None
-if not self.background or 0 in fd_pipes:
-# Subclasses such as AbstractEbuildProcess may have already passed
-# in a null file descriptor in fd_pipes, so use that when given.
-pass
+if fd_pipes or self.logfile:
+master_fd, slave_fd = self._pipe(fd_pipes)
+
+can_log = self._can_log(slave_fd)
+if can_log:
+log_file_path = self.logfile
+else:
+log_file_path = None
+
+null_input = None
+if not self.background or 0 in fd_pipes:
+# Subclasses such as AbstractEbuildProcess may have already
passed
+# in a null file descriptor in fd_pipes, so use that when
given.
+pass
+else:
+# TODO: Use job control functions like tcsetpgrp() to control
+# access to stdin. Until then, use /dev/null so that any
+# attempts to read from stdin will immediately return EOF
+# instead of blocking indefinitely.
+null_input = os.open("/dev/null", os.O_RDWR)
+fd_pipes[0] = null_input
+
+fd_pipes.setdefault(0, portage._get_stdin().fileno())
+fd_pipes.setdefault(1, sys.__stdout__.fileno())
+fd_pipes.setdefault(2, sys.__stderr__.fileno())
+
+# flush any pending output
+stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
+for fd in fd_pipes.values():
+if fd in stdout_filenos:
+sys.__stdout__.flush()
+sys.__stderr__.flush()
+break
+
+fd_pipes_orig = fd_pipes.copy()
+
+if log_file_path is not None or self.background:
+fd_pipes[1] = slave_fd
+fd_pipes[2] = slave_fd
+
+else:
+# Create a dummy pipe that PipeLogger uses to efficiently
+# monitor for process exit by listening for the EOF event.
+# Re-use of the allocated fd number for the key in fd_pipes
+# guarantees that the keys will not collide for similarly
+# allocated pipes which are used by callers such as
+# FileDigester and MergeProcess. See the _setup_pipes
+# docstring for more benefits of this allocation approach.
+self._dummy_pipe_fd = slave_fd
+fd_pipes[slave_fd] = slave_fd
else:
-# TODO: Use job control functions like tcsetpgrp() to control
-# access to stdin. Until then, use /dev/null so that any
-# attempts to read from stdin will immediately return EOF
-# instead of blocking indefinitely.
-null_input = os.open("/dev/null", os.O_RDWR)
-fd_pipes[0] = null_input
-
-fd_pipes.setdefault(0, portage._get_stdin().fileno())
-fd_pipes.setdefault(1, sys.__stdout__.fileno())
-fd_pipes.setdefault(2, sys.__stderr__.fileno())
-
-# flush any pending output
-stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
-for fd in fd_pipes.values():
-if fd in stdout_filenos:
-sys.__stdout__.flush()
-sys.__stderr__.flush()
-break
-
-fd_pipes_orig = fd_pipes.copy()
-
-if log_file_path is not None or self.background:
-
[gentoo-commits] proj/portage:master commit in: /, lib/_emerge/
commit: 7b855cb2bb50c59d5310f9e0326e8c49e4f29111
Author: Florian Schmaus gentoo org>
AuthorDate: Mon Oct 2 10:21:10 2023 +
Commit: Sam James gentoo org>
CommitDate: Mon Oct 2 12:22:24 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7b855cb2
depgraph: show backtrack information after dep resolution
To improve user's awareness of the backtracking performed by portage
and the related limit, show briefly how much backtracking was done.
Signed-off-by: Florian Schmaus gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1107
Signed-off-by: Sam James gentoo.org>
NEWS| 2 ++
lib/_emerge/depgraph.py | 21 +++--
2 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/NEWS b/NEWS
index bfa6261bb9..75680fce18 100644
--- a/NEWS
+++ b/NEWS
@@ -17,6 +17,8 @@ Features:
* emerge: Use appropriate colors if binpkgs are used in e.g. pkg_pretend
messages as well as elog's mod_echo module (bug #914159).
+* Show backtrack information after dependency resolution.
+
Bug fixes:
* Prevent gpg from removing /dev/null when unlocking signing key (bug #912808).
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index ef7dd54052..a0d69ff305 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -11401,7 +11401,7 @@ def _spinner_start(spinner, myopts):
spinner.start_time = time.time()
-def _spinner_stop(spinner):
+def _spinner_stop(spinner, backtracked: int = -1, max_retries: int = -1):
if spinner is None or spinner.update == spinner.update_quiet:
return
@@ -11414,7 +11414,14 @@ def _spinner_stop(spinner):
stop_time = time.time()
time_fmt = f"{stop_time - spinner.start_time:.2f}"
-portage.writemsg_stdout(f"Dependency resolution took {darkgreen(time_fmt)}
s.\n\n")
+
+backtrack_info = ""
+if backtracked >= 0:
+backtrack_info = f" (backtrack: {backtracked}/{max_retries})"
+
+portage.writemsg_stdout(
+f"Dependency resolution took {darkgreen(time_fmt)}
s{backtrack_info}.\n\n"
+)
def backtrack_depgraph(
@@ -11430,13 +11437,15 @@ def backtrack_depgraph(
Raises PackageSetNotFound if myfiles contains a missing package set.
"""
+backtracked, max_retries = -1, -1
_spinner_start(spinner, myopts)
try:
-return _backtrack_depgraph(
+success, mydepgraph, favorites, backtracked, max_retries =
_backtrack_depgraph(
settings, trees, myopts, myparams, myaction, myfiles, spinner
)
+return (success, mydepgraph, favorites)
finally:
-_spinner_stop(spinner)
+_spinner_stop(spinner, backtracked, max_retries)
def _backtrack_depgraph(
@@ -11447,7 +11456,7 @@ def _backtrack_depgraph(
myaction: Optional[str],
myfiles: list[str],
spinner: "_emerge.stdout_spinner.stdout_spinner",
-) -> tuple[Any, depgraph, list[str]]:
+) -> tuple[Any, depgraph, list[str], int, int]:
debug = "--debug" in myopts
mydepgraph = None
max_retries = myopts.get("--backtrack", 10)
@@ -11540,7 +11549,7 @@ def _backtrack_depgraph(
)
success, favorites = mydepgraph.select_files(myfiles)
-return (success, mydepgraph, favorites)
+return (success, mydepgraph, favorites, backtracked, max_retries)
def resume_depgraph(
[gentoo-commits] proj/portage:master commit in: /, lib/_emerge/
commit: 2b0a904f6b3e0376d0ee2359469d8732c11e376b
Author: Mike Gilbert gentoo org>
AuthorDate: Sun Oct 1 21:36:23 2023 +
Commit: Sam James gentoo org>
CommitDate: Mon Oct 2 11:47:53 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2b0a904f
search: fix 'mlen' counter for ambiguous package names
self.mlen was being incremented in both addCP() and output().
Drop the increment in addCP(), and convert mlen to a function-local
variable in output().
Bug: https://bugs.gentoo.org/915054
Signed-off-by: Mike Gilbert gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1106
Signed-off-by: Sam James gentoo.org>
NEWS | 3 +++
lib/_emerge/search.py | 14 +++---
2 files changed, 6 insertions(+), 11 deletions(-)
diff --git a/NEWS b/NEWS
index aa2b39ee9b..bfa6261bb9 100644
--- a/NEWS
+++ b/NEWS
@@ -28,6 +28,9 @@ Bug fixes:
* dispatch-conf, etc-update: fix SELinux security labels on merged files.
+* emerge: fix application count when listing search results for ambiguous
+ packages (bug #915054).
+
Cleanups:
* vartree: Remove unused variables and parameters
diff --git a/lib/_emerge/search.py b/lib/_emerge/search.py
index e422434a72..62406258d0 100644
--- a/lib/_emerge/search.py
+++ b/lib/_emerge/search.py
@@ -57,7 +57,6 @@ class search:
self.fuzzy = fuzzy
self.search_similarity = 80 if search_similarity is None else
search_similarity
self.matches = {"pkg": []}
-self.mlen = 0
self._dbs = []
@@ -377,7 +376,6 @@ class search:
if not self._xmatch("match-all", cp):
return
self.matches["pkg"].append(cp)
-self.mlen += 1
def output(self):
"""Outputs the results of the search."""
@@ -407,8 +405,9 @@ class search:
# Do a normal search
iterator = self._iter_search()
+mlen = 0
for mtype, match in iterator:
-self.mlen += 1
+mlen += 1
masked = False
full_package = None
if mtype in ("pkg", "desc"):
@@ -521,14 +520,7 @@ class search:
" " + darkgreen("License:") + " " + license
+ "\n\n"
)
-msg.append("[ Applications found : " + bold(str(self.mlen)) + " ]\n\n")
-
-# This method can be called multiple times, so
-# reset the match count for the next call. Don't
-# reset it at the beginning of this method, since
-# that would lose modfications from the addCP
-# method.
-self.mlen = 0
+msg.append("[ Applications found : " + bold(str(mlen)) + " ]\n\n")
#
# private interface
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: c6ee4348df65210ee4c052b894cbc9f4f54f6ce8 Author: Daniel Harding living180 net> AuthorDate: Sun Sep 10 06:51:13 2023 + Commit: Sam James gentoo org> CommitDate: Thu Sep 21 15:46:00 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c6ee4348 depgraph: improve reverse dep caching Avoid applying @functools.lru_cache() to the depgraph._slot_operator_check_reverse_depencencies() method, as applying the lru_cache decorator to a method can prevent the class instances from being garbage collected due to self being stored in the cache as part of the args key[1]. Instead, set up a per-instance cache in the __init__() method. [1] https://stackoverflow.com/q/33672412/3573385 Bug: https://bugs.gentoo.org/883071 Signed-off-by: Daniel Harding living180.net> Signed-off-by: Sam James gentoo.org> lib/_emerge/depgraph.py | 9 - 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 01a49bcb53..0629acab22 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -679,6 +679,12 @@ class depgraph: self.query = UserQuery(myopts).query +# Set up a per-instance memoization cache for the +# _slot_operator_check_reverse_dependencies() method: +self._slot_operator_check_reverse_dependencies = functools.lru_cache( +maxsize=100 +)(self._slot_operator_check_reverse_dependencies) + def _index_binpkgs(self): for root in self._frozen_config.trees: bindb = self._frozen_config.trees[root]["bintree"].dbapi @@ -2214,7 +2220,8 @@ class depgraph: return None -@functools.lru_cache(maxsize=100) +# This method is memoized on a per-instance basis via a decorator applied +# in __init__(). def _slot_operator_check_reverse_dependencies( self, existing_pkg, candidate_pkg, replacement_parent=None ):
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 53236c55ba8c8106afc80fa113a876ae5784f1ed Author: Daniel Harding living180 net> AuthorDate: Thu Sep 21 07:42:50 2023 + Commit: Sam James gentoo org> CommitDate: Thu Sep 21 15:46:04 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=53236c55 depgraph: increase reverse dep cache size To avoid blowing out the cache and destroying performance, increase the maximum size of the memoization cache for the depgraph _slot_opererator_check_reverse_dependencies() method from 100 to 1000. Bug: https://bugs.gentoo.org/883071 Signed-off-by: Daniel Harding living180.net> Closes: https://github.com/gentoo/portage/pull/1092 Signed-off-by: Sam James gentoo.org> lib/_emerge/depgraph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 0629acab22..ef7dd54052 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -682,7 +682,7 @@ class depgraph: # Set up a per-instance memoization cache for the # _slot_operator_check_reverse_dependencies() method: self._slot_operator_check_reverse_dependencies = functools.lru_cache( -maxsize=100 +maxsize=1000 )(self._slot_operator_check_reverse_dependencies) def _index_binpkgs(self):
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 7280f1b3dc19ae1be1cc7a930346127b1b7e7686
Author: Sam James gentoo org>
AuthorDate: Fri Sep 15 03:38:13 2023 +
Commit: Sam James gentoo org>
CommitDate: Fri Sep 15 10:36:47 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7280f1b3
_emerge: use binpkg coloring for pkg_pretend
For the 'Running pre-merge checks' message, we write:
```
>>> Running pre-merge checks for X
```
X is currently always in green, while the emerge list above might have the atom
X listed in purple if it's a binpkg.
Change X to be colored based on if it's a binpkg or not.
Bug: https://bugs.gentoo.org/914159
Signed-off-by: Sam James gentoo.org>
lib/_emerge/Scheduler.py | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index ece3f27f7c..db9615d406 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -865,12 +865,13 @@ class Scheduler(PollScheduler):
if self._terminated_tasks:
raise asyncio.CancelledError
-out_str = "Running pre-merge checks for " + colorize("INFORM",
x.cpv)
-self._status_msg(out_str)
-
root_config = x.root_config
settings = self._allocate_config(root_config.root)
settings.setcpv(x)
+
+color = "PKG_BINARY_MERGE" if x.built else "INFORM"
+self._status_msg(f"Running pre-merge checks for {colorize(color,
x.cpv)}")
+
if not x.built:
# Get required SRC_URI metadata (it's not cached in x.metadata
# because some packages have an extremely large SRC_URI value).
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: f5081b7def54bc9e49aac117ce2b1ebc89379c79
Author: Sam James gentoo org>
AuthorDate: Fri Sep 15 04:00:32 2023 +
Commit: Sam James gentoo org>
CommitDate: Fri Sep 15 10:36:47 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f5081b7d
_emerge: use binpkg coloring for failures too
Bug: https://bugs.gentoo.org/914159
Signed-off-by: Sam James gentoo.org>
lib/_emerge/Scheduler.py | 9 +++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index db9615d406..2d1872feba 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -1260,12 +1260,14 @@ class Scheduler(PollScheduler):
and not mod_echo_output
):
for mysettings, key, logentries in self._failed_pkgs_die_msgs:
+color = "PKG_BINARY_MERGE" if pkg.built else "INFORM"
+
root_msg = ""
if mysettings["ROOT"] != "/":
root_msg = f" merged to {mysettings['ROOT']}"
print()
printer.einfo(
-f"Error messages for package {colorize('INFORM',
key)}{root_msg}:"
+f"Error messages for package {colorize(color,
key)}{root_msg}:"
)
print()
for phase in portage.const.EBUILD_PHASES:
@@ -2001,7 +2003,10 @@ class Scheduler(PollScheduler):
def _failed_pkg_msg(self, failed_pkg, action, preposition):
pkg = failed_pkg.pkg
-msg = f"{bad('Failed')} to {action} {colorize('INFORM', pkg.cpv)}"
+
+color = "PKG_BINARY_MERGE" if failed_pkg.pkg.built else "INFORM"
+
+msg = f"{bad('Failed')} to {action} {colorize(color, pkg.cpv)}"
if pkg.root_config.settings["ROOT"] != "/":
msg += f" {preposition} {pkg.root}"
[gentoo-commits] proj/portage:master commit in: /, lib/_emerge/
commit: ab71ec960b65b2d77e5669d8979dc72e3f4ffb41 Author: Sam James gentoo org> AuthorDate: Thu Aug 17 06:46:53 2023 + Commit: Sam James gentoo org> CommitDate: Thu Aug 17 06:52:55 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ab71ec96 lib: _emerge: call portage.util.initialize_logger() We don't have a root logger setup otherwise which means we can't get nice output from things like gemato which define their own logger names/domains. And respect --debug for it. Signed-off-by: Sam James gentoo.org> NEWS| 9 + lib/_emerge/main.py | 4 2 files changed, 13 insertions(+) diff --git a/NEWS b/NEWS index d442a42ab0..ad541b95b1 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,12 @@ +portage-3.0.51 (UNRELEASED) +-- + +Features: +* emerge: Set up logging immediately and respect --debug for log level. + +* sync: git, rsync: now respects --debug for better output from gemato. This is + especially useful for debugging hangs during the 'Refreshing keys' stage. + portage-3.0.50 (2023-08-09) -- diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py index 41bed09696..d92f069901 100644 --- a/lib/_emerge/main.py +++ b/lib/_emerge/main.py @@ -1191,6 +1191,10 @@ def emerge_main(args: Optional[list[str]] = None): myaction, myopts, myfiles = parse_opts(args, silent=True) if "--debug" in myopts: os.environ["PORTAGE_DEBUG"] = "1" +portage.util.initialize_logger(logging.DEBUG) +else: +portage.util.initialize_logger() + if "--config-root" in myopts: os.environ["PORTAGE_CONFIGROOT"] = myopts["--config-root"] if "--sysroot" in myopts:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: f228252f4b4c3b33ff1e199f55bec9a6a104b80c Author: Berin Aniesh gmail com> AuthorDate: Mon Jul 10 03:10:07 2023 + Commit: Sam James gentoo org> CommitDate: Tue Jul 11 05:00:43 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f228252f depgraph.py: fix wrong type annotations Fixes: 7e8a730b85b77f9495f4999ac173dc726777203f Bug: https://bugs.gentoo.org/910171 Signed-off-by: Berin Aniesh gmail.com> Closes: https://github.com/gentoo/portage/pull/1067 Signed-off-by: Sam James gentoo.org> lib/_emerge/depgraph.py | 18 +++--- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 1aeae6257..72747d7bc 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -101,7 +101,11 @@ from _emerge.resolver.circular_dependency import circular_dependency_handler from _emerge.resolver.output import Display, format_unmatched_atom # Type annotation imports -from typing import Any, Optional, Dict, List, Tuple, Union +from typing import Any, Optional, Dict, List, Tuple, Union, TYPE_CHECKING + +if TYPE_CHECKING: +import _emerge.stdout_spinner.stdout_spinner + # Exposes a depgraph interface to dep_check. _dep_check_graph_interface = collections.namedtuple( @@ -11409,7 +11413,7 @@ def _spinner_stop(spinner): def backtrack_depgraph( settings: portage.package.ebuild.config.config, trees: portage._trees_dict, -myopts: Dict[str, Any], +myopts: Dict[str, Union[str, int, bool]], myparams: Dict[str, Union[int, str, bool]], myaction: Optional[str], myfiles: List[str], @@ -11431,7 +11435,7 @@ def backtrack_depgraph( def _backtrack_depgraph( settings: portage.package.ebuild.config.config, trees: portage._trees_dict, -myopts: Dict[str, Any], +myopts: Dict[str, Union[str, int, bool]], myparams: Dict[str, Union[int, str, bool]], myaction: Optional[str], myfiles: List[str], @@ -11536,8 +11540,8 @@ def resume_depgraph( settings: portage.package.ebuild.config.config, trees: portage._trees_dict, mtimedb: Any, -myopts: Dict[str, str], -myparams: Dict[str, Any], +myopts: Dict[str, Union[str, int, bool]], +myparams: Dict[str, Union[str, bool]], spinner: "_emerge.stdout_spinner.stdout_spinner", ): """ @@ -11554,8 +11558,8 @@ def _resume_depgraph( settings: portage.package.ebuild.config.config, trees: portage._trees_dict, mtimedb: Any, -myopts: Dict[str, str], -myparams: Dict[str, Any], +myopts: Dict[str, Union[str, int, bool]], +myparams: Dict[str, Union[str, bool]], spinner: "_emerge.stdout_spinner.stdout_spinner", ): """
[gentoo-commits] proj/portage:master commit in: /, lib/_emerge/
commit: b199d0307b47f9fb06dbe533d7e24926a561c238
Author: Oskari Pirhonen gmail com>
AuthorDate: Sun Jul 9 05:08:27 2023 +
Commit: Sam James gentoo org>
CommitDate: Sun Jul 9 06:24:28 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b199d030
depgraph.py: fix "no ebuilds/binpkgs" message
The "there are no binary packages to satisfy" was being unconditionally
output for packages that could not be found. Fix the logic for choosing
between the "binary packages" and "ebuilds" form of the message.
This is a temporary stopgap as alluded to by me in the bug, but the
tl;dr is that some entries in the `myopts` dict have "y"/"n" values
whereas some are True/unset, and this discrepancy should be sorted out.
[sam: Add NEWS and Fixes, although the change in that commit _shouldn't_
have been wrong, it is because of a quirk for now...]
Bug: https://bugs.gentoo.org/909853
Signed-off-by: Oskari Pirhonen gmail.com>
Closes: https://github.com/gentoo/portage/pull/1065
Fixes: 0b21a5a392bd84c07b94373991f59108fbe98516
Signed-off-by: Sam James gentoo.org>
NEWS| 3 +++
lib/_emerge/depgraph.py | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/NEWS b/NEWS
index d40c309cd..fe59a1ddf 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,9 @@ Bug fixes:
* install-qa-check.d/05prefix: Fix prefixifying shebang for >= EAPI 7 ebuilds
(bug #909147).
+* emerge: Fix 'no ebuilds available' message always mentioning binpkgs
+ (bug #909853).
+
* gpkg: Fix timestamp for binary packages (bug #909067).
portage-3.0.49 (2023-06-21)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index a36ab6351..1aeae6257 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -6456,7 +6456,7 @@ class depgraph:
cp_exists = True
break
-if self._frozen_config.myopts.get("--usepkgonly", "y"):
+if self._frozen_config.myopts.get("--usepkgonly", False):
writemsg(
f"\nemerge: there are no binary packages to satisfy
{green(xinfo)}.\n",
noiselevel=-1,
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 043ce446b6451fc9e74539634fc2e8e266c8b39e
Author: Berin Aniesh gmail com>
AuthorDate: Mon Jun 19 01:32:12 2023 +
Commit: Sam James gentoo org>
CommitDate: Thu Jun 29 08:19:28 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=043ce446
actions.py: Remove unused variable
Signed-off-by: Berin Aniesh gmail.com>
Closes: https://github.com/gentoo/portage/pull/1058
Signed-off-by: Sam James gentoo.org>
lib/_emerge/actions.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 74760..0fee21459 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1908,7 +1908,6 @@ def action_info(settings, trees, myopts, myfiles):
append(
getportageversion(
settings["PORTDIR"],
-None,
settings.profile_path,
chost,
trees[settings["EROOT"]]["vartree"].dbapi,
@@ -2858,7 +2857,6 @@ def get_profile_version(
def getportageversion(
portdir: str,
-_unused: None,
profile: str,
chost: str,
vardb: portage.dbapi.vartree.vardbapi,
@@ -3527,7 +3525,6 @@ def run_action(emerge_config):
writemsg_stdout(
getportageversion(
emerge_config.target_config.settings["PORTDIR"],
-None,
emerge_config.target_config.settings.profile_path,
emerge_config.target_config.settings.get("CHOST"),
emerge_config.target_config.trees["vartree"].dbapi,
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 5de6c63d0779eee0942a294b3a53ee1cce9d6d07
Author: Berin Aniesh gmail com>
AuthorDate: Sun Jun 18 03:41:12 2023 +
Commit: Sam James gentoo org>
CommitDate: Thu Jun 29 08:19:27 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5de6c63d
actions.py: refactor getportageversion() into smaller functions
Signed-off-by: Berin Aniesh gmail.com>
Signed-off-by: Sam James gentoo.org>
lib/_emerge/actions.py | 51 --
1 file changed, 29 insertions(+), 22 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 60199dae6..dae6b6b7b 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -2789,16 +2789,23 @@ def relative_profile_path(portdir, abs_profile):
profilever = None
return profilever
+def get_libc_version(vardb):
+libcver = []
+libclist = set()
+for atom in expand_new_virt(vardb, portage.const.LIBC_PACKAGE_ATOM):
+if not atom.blocker:
+libclist.update(vardb.match(atom))
+if libclist:
+for cpv in sorted(libclist):
+libc_split = portage.catpkgsplit(cpv)[1:]
+if libc_split[-1] == "r0":
+libc_split = libc_split[:-1]
+libcver.append("-".join(libc_split))
+else:
+libcver = ["unavailable"]
+return libcver
-def getportageversion(portdir, _unused, profile, chost, vardb):
-pythonver = (
-"python"
-f" {sys.version_info[0]}"
-f".{sys.version_info[1]}"
-f".{sys.version_info[2]}"
-f"-{sys.version_info[3]}"
-f"-{sys.version_info[4]}"
-)
+def get_profile_version(portdir, profile, vardb):
profilever = None
repositories = vardb.settings.repositories
if profile:
@@ -2838,21 +2845,21 @@ def getportageversion(portdir, _unused, profile, chost,
vardb):
if profilever is None:
profilever = "unavailable"
+
+return profilever
-libcver = []
-libclist = set()
-for atom in expand_new_virt(vardb, portage.const.LIBC_PACKAGE_ATOM):
-if not atom.blocker:
-libclist.update(vardb.match(atom))
-if libclist:
-for cpv in sorted(libclist):
-libc_split = portage.catpkgsplit(cpv)[1:]
-if libc_split[-1] == "r0":
-libc_split = libc_split[:-1]
-libcver.append("-".join(libc_split))
-else:
-libcver = ["unavailable"]
+def getportageversion(portdir, _unused, profile, chost, vardb):
+pythonver = (
+"python"
+f" {sys.version_info[0]}"
+f".{sys.version_info[1]}"
+f".{sys.version_info[2]}"
+f"-{sys.version_info[3]}"
+f"-{sys.version_info[4]}"
+)
+profilever = get_profile_version(portdir, profile, vardb)
+libcver = get_libc_version(vardb)
gccver = getgccversion(chost)
unameout = platform.release() + " " + platform.machine()
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: e29cb58272b26fb6a4eeae27f4b127f80b34a88c
Author: Berin Aniesh gmail com>
AuthorDate: Mon Jun 19 01:30:25 2023 +
Commit: Sam James gentoo org>
CommitDate: Thu Jun 29 08:19:28 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e29cb582
actions.py: More type annotations
Signed-off-by: Berin Aniesh gmail.com>
Signed-off-by: Sam James gentoo.org>
lib/_emerge/actions.py | 17 ++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 01bf82fcb..74760 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -105,6 +105,9 @@ from _emerge.UnmergeDepPriority import UnmergeDepPriority
from _emerge.UseFlagDisplay import pkg_use_display
from _emerge.UserQuery import UserQuery
+# Type annotation imports
+from typing import List
+
def action_build(
emerge_config,
@@ -2790,7 +2793,7 @@ def relative_profile_path(portdir, abs_profile):
return profilever
-def get_libc_version(vardb):
+def get_libc_version(vardb: portage.dbapi.vartree.vardbapi) -> List[str]:
libcver = []
libclist = set()
for atom in expand_new_virt(vardb, portage.const.LIBC_PACKAGE_ATOM):
@@ -2807,7 +2810,9 @@ def get_libc_version(vardb):
return libcver
-def get_profile_version(portdir, profile, vardb):
+def get_profile_version(
+portdir: str, profile: str, vardb: portage.dbapi.vartree.vardbapi
+) -> str:
profilever = None
repositories = vardb.settings.repositories
if profile:
@@ -2851,7 +2856,13 @@ def get_profile_version(portdir, profile, vardb):
return profilever
-def getportageversion(portdir, _unused, profile, chost, vardb):
+def getportageversion(
+portdir: str,
+_unused: None,
+profile: str,
+chost: str,
+vardb: portage.dbapi.vartree.vardbapi,
+) -> str:
pythonver = (
"python"
f" {sys.version_info[0]}"
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 224207c7d1988a354d004507bb7ecfb90b4ef097 Author: YiFei Zhu gmail com> AuthorDate: Tue Jun 13 00:47:52 2023 + Commit: Sam James gentoo org> CommitDate: Fri Jun 16 03:34:46 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=224207c7 depgraph: Do not allow slotted deps to be satisfied by wrong slots This may be part of what caused the "perl rebuild bug". The "priority" of perl, when seen by perl modules, may have "satisfied" set to an installed perl of a wrong slot. Compounding this factor with bug #756199 where find_smallest_cycle would select a single node, in this case because the dependency of perl is satisfied and the priority then gets ignored, the "cycle" becomes a perl module alone and gets rebuilt early. I also updated the test from the previous patch to account for this change. No other tests seems affected. For a larger scale test, I reproduced this initially with a stage3 chroot from a Jan 1 2022 stage3 snapshot, and testing in an equivalent dockerfile would work too: FROM gentoo/stage3:amd64-openrc-20220101 RUN emerge-webrsync COPY . /portage Before this patch (USE flags omitted): # cd /portage && bin/emerge -puDN @world 2>&1 | grep -i perl [ebuild U ] app-admin/perl-cleaner-2.30-r1 [2.30] [ebuild rR] virtual/perl-File-Temp-0.231.100 [ebuild rR] dev-perl/Locale-gettext-1.70.0-r1 [ebuild rR] dev-perl/MIME-Charset-1.12.2-r1 [ebuild rR] dev-perl/Module-Build-0.423.100 [ebuild rR] dev-perl/Text-CharWidth-0.40.0-r2 [ebuild U ] dev-lang/perl-5.36.0-r2 [5.34.0-r3] [ebuild N ] virtual/perl-CPAN-2.330.0 [ebuild U ] virtual/perl-ExtUtils-MakeMaker-7.640.0 [7.620.0] [ebuild U ] virtual/perl-File-Spec-3.840.0 [3.800.0] [...] After this patch: # cd /portage && bin/emerge -puDN @world 2>&1 | grep -i perl [ebuild U ] app-admin/perl-cleaner-2.30-r1 [2.30] [ebuild U ] dev-lang/perl-5.36.0-r2:0/5.36 [5.34.0-r3:0/5.34] [ebuild N ] virtual/perl-CPAN-2.330.0 [ebuild U ] virtual/perl-ExtUtils-MakeMaker-7.640.0 [7.620.0] [ebuild U ] virtual/perl-Data-Dumper-2.184.0 [2.179.0] [ebuild U ] virtual/perl-File-Spec-3.840.0 [3.800.0] [ebuild U ] virtual/perl-Test-Harness-3.440.0-r1 [3.430.0] [ebuild rR] dev-perl/Pod-Parser-1.630.0-r1 [ebuild rR] dev-perl/Text-CharWidth-0.40.0-r2 [ebuild rR] dev-perl/Text-WrapI18N-0.60.0-r2 [...] Bug: https://bugs.gentoo.org/463976 Bug: https://bugs.gentoo.org/592880 Bug: https://bugs.gentoo.org/596664 Bug: https://bugs.gentoo.org/631490 Bug: https://bugs.gentoo.org/764365 Bug: https://bugs.gentoo.org/793992 Signed-off-by: YiFei Zhu gmail.com> Closes: https://github.com/gentoo/portage/pull/1055 Signed-off-by: Sam James gentoo.org> lib/_emerge/depgraph.py | 18 ++ lib/portage/tests/resolver/test_perl_rebuild_bug.py | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 60e57b226..28acfed9d 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -4042,6 +4042,16 @@ class depgraph: inst_pkg, modified_use=self._pkg_use_enabled(inst_pkg) ) ] +# Do not allow slotted deps to be satisfied by wrong slots. +# Otherwise, slot-operator-dependent packages may rebuild +# before the slotted package they are dependent on. +if child and atom.slot_operator == "=": +inst_pkgs = [ +inst_pkg +for inst_pkg in inst_pkgs +if inst_pkg.slot == child.slot +and inst_pkg.sub_slot == child.sub_slot +] if inst_pkgs: for inst_pkg in inst_pkgs: if self._pkg_visibility_check(inst_pkg): @@ -4161,6 +4171,14 @@ class depgraph: inst_pkg, modified_use=self._pkg_use_enabled(inst_pkg) ) ] +# Do not allow slotted deps to be satisfied by wrong slots. +if child and atom.slot_operator == "=": +inst_pkgs = [ +inst_pkg +for inst_pkg in inst_pkgs +if inst_pkg.slot == child.slot +and inst_pkg.sub_slot == child.sub_slot +] if inst_pkgs: for inst_pkg in inst_pkgs: if self._pkg_visibility_check(inst_pkg): diff --git a/lib/portage/tests/resolver/test_perl_rebuild_bug.py b/lib/portage/tests/resolver/test_perl_rebuild_bug.py index 928fd47d7..7e376f396 100644 --- a/lib/portage/tests/resolver/test_perl_rebuild_bug.py +++
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 44afa8445dc46464200fe46c1e09e0c7475067bf
Author: YiFei Zhu gmail com>
AuthorDate: Mon Jun 12 02:23:09 2023 +
Commit: Sam James gentoo org>
CommitDate: Fri Jun 16 03:34:46 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=44afa844
depgraph: Don't ignore downgrades as missed_updates
Missed updates can also come in the form of package downgrades,
when, for example, there are keyword changes. They can cause
rebuilds, and these rebuilds may be not possible due to reasons such
as keywords or masks. In this case, prior to this patch, portage
would cancel the downgrade, but the rebuilds would be requested
endlessly, because bug 439688's backtrack code does not trigger.
To reproduce, on an ACCEPT_KEYWORDS=~amd64 machine, emerge
=dev-libs/openssl=3.0.9, dev-util/rustup, and something else that
depends on openssl. Then un-accept ~amd64 for openssl and rustup.
Prior to this patch, a @world upgrade would cause:
These are the packages that would be merged, in order:
Calculating dependencies... done!
[ebuild rR] dev-libs/libevent-2.1.12-r1:0/2.1-7::gentoo
[ebuild rR] net-misc/rsync-3.2.7-r2::gentoo
[...]
Total: 71 packages (71 reinstalls), Size of downloads: 0 KiB
There are no packages marked "R", only "rR". There are no section
labeled "The following packages are causing rebuilds:" either.
After this patch, we have:
These are the packages that would be merged, in order:
Calculating dependencies... done!
Total: 0 packages, Size of downloads: 0 KiB
WARNING: One or more updates/rebuilds have been skipped due to a dependency
conflict:
dev-libs/openssl:0
(dev-libs/openssl-1.1.1u:0/1.1::gentoo, ebuild scheduled for merge)
dev-libs/openssl:0/3= required by (dev-util/rustup-1.25.2:0/0::gentoo,
installed)
I also updated the test from the previous patch to account for
this change. No other tests seems affected.
Bug: https://bugs.gentoo.org/439688
Bug: https://bugs.gentoo.org/622270
Signed-off-by: YiFei Zhu gmail.com>
Closes: https://github.com/gentoo/portage/pull/1053
Signed-off-by: Sam James gentoo.org>
lib/_emerge/depgraph.py| 4 +---
lib/portage/tests/resolver/test_slot_conflict_blocked_prune.py | 2 +-
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 77133e99c..60e57b226 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1287,9 +1287,7 @@ class depgraph:
pkg.root, pkg.slot_atom
):
any_selected = True
-if chosen_pkg > pkg or (
-not chosen_pkg.installed and chosen_pkg.version ==
pkg.version
-):
+if not chosen_pkg.installed and chosen_pkg.version ==
pkg.version:
missed_update = False
break
if any_selected and missed_update:
diff --git a/lib/portage/tests/resolver/test_slot_conflict_blocked_prune.py
b/lib/portage/tests/resolver/test_slot_conflict_blocked_prune.py
index 14e98cd00..b23126d5f 100644
--- a/lib/portage/tests/resolver/test_slot_conflict_blocked_prune.py
+++ b/lib/portage/tests/resolver/test_slot_conflict_blocked_prune.py
@@ -63,7 +63,7 @@ class SlotConflictBlockedPruneTestCase(TestCase):
["@world"],
options={"--deep": True, "--update": True, "--verbose": True},
success=True,
-mergelist=["x11-base/xwayland-23.1.1"],
+mergelist=[],
),
)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: a5b4eb5a69be4b0abe0c0ee682c40f1881be084f Author: Sam James gentoo org> AuthorDate: Wed Jun 14 05:06:25 2023 + Commit: Sam James gentoo org> CommitDate: Wed Jun 14 05:06:25 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a5b4eb5a depgraph: drop unnecessary spinner import Signed-off-by: Sam James gentoo.org> lib/_emerge/depgraph.py | 9 - 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 94a4fff3e..77133e99c 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -102,7 +102,6 @@ from _emerge.resolver.output import Display, format_unmatched_atom # Type annotation imports from typing import Any, Optional, Dict, List, Tuple, Union -from _emerge.stdout_spinner import stdout_spinner # Exposes a depgraph interface to dep_check. _dep_check_graph_interface = collections.namedtuple( @@ -11396,7 +11395,7 @@ def backtrack_depgraph( myparams: Dict[str, Union[int, str, bool]], myaction: Optional[str], myfiles: List[str], -spinner: stdout_spinner, +spinner: "_emerge.stdout_spinner.stdout_spinner", ) -> Tuple[Any, depgraph, List[str]]: """ @@ -11418,7 +11417,7 @@ def _backtrack_depgraph( myparams: Dict[str, Union[int, str, bool]], myaction: Optional[str], myfiles: List[str], -spinner: stdout_spinner, +spinner: "_emerge.stdout_spinner.stdout_spinner", ) -> Tuple[Any, depgraph, List[str]]: debug = "--debug" in myopts mydepgraph = None @@ -11521,7 +11520,7 @@ def resume_depgraph( mtimedb: Any, myopts: Dict[str, str], myparams: Dict[str, Any], -spinner: stdout_spinner, +spinner: "_emerge.stdout_spinner.stdout_spinner", ): """ Raises PackageSetNotFound if myfiles contains a missing package set. @@ -11539,7 +11538,7 @@ def _resume_depgraph( mtimedb: Any, myopts: Dict[str, str], myparams: Dict[str, Any], -spinner: stdout_spinner, +spinner: "_emerge.stdout_spinner.stdout_spinner", ): """ Construct a depgraph for the given resume list. This will raise
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 7e8a730b85b77f9495f4999ac173dc726777203f
Author: Berin Aniesh gmail com>
AuthorDate: Mon Jun 12 03:56:25 2023 +
Commit: Sam James gentoo org>
CommitDate: Wed Jun 14 05:03:27 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7e8a730b
depgraph: type annotations
Signed-off-by: Berin Aniesh gmail.com>
Closes: https://github.com/gentoo/portage/pull/1054
Signed-off-by: Sam James gentoo.org>
lib/_emerge/depgraph.py | 43 +++
1 file changed, 39 insertions(+), 4 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 861579a6d..94a4fff3e 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -100,6 +100,10 @@ from _emerge.resolver.slot_collision import
slot_conflict_handler
from _emerge.resolver.circular_dependency import circular_dependency_handler
from _emerge.resolver.output import Display, format_unmatched_atom
+# Type annotation imports
+from typing import Any, Optional, Dict, List, Tuple, Union
+from _emerge.stdout_spinner import stdout_spinner
+
# Exposes a depgraph interface to dep_check.
_dep_check_graph_interface = collections.namedtuple(
"_dep_check_graph_interface",
@@ -11385,8 +11389,17 @@ def _spinner_stop(spinner):
portage.writemsg_stdout(f"Dependency resolution took {darkgreen(time_fmt)}
s.\n\n")
-def backtrack_depgraph(settings, trees, myopts, myparams, myaction, myfiles,
spinner):
+def backtrack_depgraph(
+settings: portage.package.ebuild.config.config,
+trees: portage._trees_dict,
+myopts: Dict[str, Any],
+myparams: Dict[str, Union[int, str, bool]],
+myaction: Optional[str],
+myfiles: List[str],
+spinner: stdout_spinner,
+) -> Tuple[Any, depgraph, List[str]]:
"""
+
Raises PackageSetNotFound if myfiles contains a missing package set.
"""
_spinner_start(spinner, myopts)
@@ -11398,7 +11411,15 @@ def backtrack_depgraph(settings, trees, myopts,
myparams, myaction, myfiles, spi
_spinner_stop(spinner)
-def _backtrack_depgraph(settings, trees, myopts, myparams, myaction, myfiles,
spinner):
+def _backtrack_depgraph(
+settings: portage.package.ebuild.config.config,
+trees: portage._trees_dict,
+myopts: Dict[str, Any],
+myparams: Dict[str, Union[int, str, bool]],
+myaction: Optional[str],
+myfiles: List[str],
+spinner: stdout_spinner,
+) -> Tuple[Any, depgraph, List[str]]:
debug = "--debug" in myopts
mydepgraph = None
max_retries = myopts.get("--backtrack", 10)
@@ -11494,7 +11515,14 @@ def _backtrack_depgraph(settings, trees, myopts,
myparams, myaction, myfiles, sp
return (success, mydepgraph, favorites)
-def resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner):
+def resume_depgraph(
+settings: portage.package.ebuild.config.config,
+trees: portage._trees_dict,
+mtimedb: Any,
+myopts: Dict[str, str],
+myparams: Dict[str, Any],
+spinner: stdout_spinner,
+):
"""
Raises PackageSetNotFound if myfiles contains a missing package set.
"""
@@ -11505,7 +11533,14 @@ def resume_depgraph(settings, trees, mtimedb, myopts,
myparams, spinner):
_spinner_stop(spinner)
-def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner):
+def _resume_depgraph(
+settings: portage.package.ebuild.config.config,
+trees: portage._trees_dict,
+mtimedb: Any,
+myopts: Dict[str, str],
+myparams: Dict[str, Any],
+spinner: stdout_spinner,
+):
"""
Construct a depgraph for the given resume list. This will raise
PackageNotFound or depgraph.UnsatisfiedResumeDep when necessary.
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: a818381d0faeec7aeefad0894df8b8fefa376dc5 Author: Sam James gentoo org> AuthorDate: Wed Jun 14 00:55:44 2023 + Commit: Sam James gentoo org> CommitDate: Wed Jun 14 00:55:44 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a818381d emerge: depgraph: fix comment typo Reported by Arfrever at https://github.com/gentoo/portage/commit/4c25c0d7af7ad71fccbfafe1e5019116c691968e#r45851837. Signed-off-by: Sam James gentoo.org> lib/_emerge/depgraph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 2c9820da1..861579a6d 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -2226,7 +2226,7 @@ class depgraph: if parent in built_slot_operator_parents: if hasattr(atom, "_orig_atom"): # If atom is the result of virtual expansion, then -# derefrence it to _orig_atom so that it will be correctly +# dereference it to _orig_atom so that it will be correctly # handled as a built slot operator dependency when # appropriate (see bug 764764). atom = atom._orig_atom
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 11b975054dc001b69f0138f1335b2d384ece184b Author: Berin Aniesh gmail com> AuthorDate: Mon May 22 10:50:02 2023 + Commit: Sam James gentoo org> CommitDate: Tue May 23 02:59:13 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=11b97505 actions: drop duplication of --help logic --help is already handled in `emerge_main()` and the deleted codeblock is not needed / unreachable. Signed-off-by: Berin Aniesh gmail.com> Closes: https://github.com/gentoo/portage/pull/1044 Signed-off-by: Sam James gentoo.org> lib/_emerge/actions.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index 0aceb2c44..60199dae6 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -3522,9 +3522,6 @@ def run_action(emerge_config): noiselevel=-1, ) return 0 -if emerge_config.action == "help": -emerge_help() -return 0 spinner = stdout_spinner() if "candy" in emerge_config.target_config.settings.features:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/_emerge/resolver/
commit: 0b21a5a392bd84c07b94373991f59108fbe98516
Author: Oskari Pirhonen gmail com>
AuthorDate: Wed Mar 22 01:56:23 2023 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 30 23:30:07 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0b21a5a3
Convert %-formats to fstrings
Files under lib/_emerge
Signed-off-by: Oskari Pirhonen gmail.com>
Closes: https://github.com/gentoo/portage/pull/1013
Signed-off-by: Sam James gentoo.org>
lib/_emerge/AbstractEbuildProcess.py | 64 -
lib/_emerge/AsynchronousLock.py| 5 +-
lib/_emerge/BinpkgEnvExtractor.py | 5 +-
lib/_emerge/BinpkgExtractorAsync.py| 60 -
lib/_emerge/BinpkgFetcher.py | 3 +-
lib/_emerge/BinpkgVerifier.py | 17 +--
lib/_emerge/EbuildMerge.py | 8 +-
lib/_emerge/JobStatusDisplay.py| 8 +-
lib/_emerge/MergeListItem.py | 4 +-
lib/_emerge/Package.py | 8 +-
lib/_emerge/Scheduler.py | 6 +-
lib/_emerge/SpawnProcess.py| 11 +-
lib/_emerge/SubProcess.py | 2 +-
lib/_emerge/Task.py| 5 +-
lib/_emerge/UserQuery.py | 5 +-
lib/_emerge/actions.py | 128 --
lib/_emerge/chk_updated_cfg_files.py | 6 +-
lib/_emerge/countdown.py | 6 +-
lib/_emerge/depgraph.py| 236 -
lib/_emerge/resolver/output.py | 3 +-
lib/_emerge/resolver/output_helpers.py | 10 +-
lib/_emerge/resolver/slot_collision.py | 65 +++--
lib/_emerge/search.py | 3 +-
lib/_emerge/unmerge.py | 28 ++--
24 files changed, 271 insertions(+), 425 deletions(-)
diff --git a/lib/_emerge/AbstractEbuildProcess.py
b/lib/_emerge/AbstractEbuildProcess.py
index fe2ef426a..be257a620 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -13,7 +13,6 @@ from _emerge.EbuildBuildDir import EbuildBuildDir
from _emerge.EbuildIpcDaemon import EbuildIpcDaemon
import portage
from portage.elog import messages as elog_messages
-from portage.localization import _
from portage.package.ebuild._ipc.ExitCommand import ExitCommand
from portage.package.ebuild._ipc.QueryCommand import QueryCommand
from portage import os
@@ -72,10 +71,10 @@ class AbstractEbuildProcess(SpawnProcess):
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and not
os.path.isdir(self.settings["PORTAGE_BUILDDIR"]):
-msg = _(
-"The ebuild phase '%s' has been aborted "
-"since PORTAGE_BUILDDIR does not exist: '%s'"
-) % (self.phase, self.settings["PORTAGE_BUILDDIR"])
+msg = (
+f"The ebuild phase '{self.phase}' has been aborted since "
+f"PORTAGE_BUILDDIR does not exist:
'{self.settings['PORTAGE_BUILDDIR']}'"
+)
self._eerror(textwrap.wrap(msg, 72))
self.returncode = 1
self._async_wait()
@@ -326,11 +325,10 @@ class AbstractEbuildProcess(SpawnProcess):
def _orphan_process_warn(self):
phase = self.phase
-msg = _(
-"The ebuild phase '%s' with pid %s appears "
-"to have left an orphan process running in the "
-"background."
-) % (phase, self.pid)
+msg = (
+f"The ebuild phase '{phase}' with pid {self.pid} appears "
+"to have left an orphan process running in the background."
+)
self._eerror(textwrap.wrap(msg, 72))
@@ -346,38 +344,32 @@ class AbstractEbuildProcess(SpawnProcess):
) or os.isatty(slave_fd)
def _killed_by_signal(self, signum):
-msg = _("The ebuild phase '%s' has been " "killed by signal %s.") % (
-self.phase,
-signum,
-)
+msg = f"The ebuild phase '{self.phase}' has been killed by signal
{signum}."
self._eerror(textwrap.wrap(msg, 72))
def _unexpected_exit(self):
phase = self.phase
msg = (
-_(
-"The ebuild phase '%s' has exited "
-"unexpectedly. This type of behavior "
-"is known to be triggered "
-"by things such as failed variable "
-"assignments (bug #190128) or bad substitution "
-"errors (bug #200313). Normally, before exiting, bash should "
-"have displayed an error message above. If bash did not "
-"produce an error message above, it's possible "
-"that the ebuild has called `exit` when it "
-"should have called `die` instead. This behavior may also "
-"be triggered by a corrupt bash binary or a hardware "
-"problem such as memory or cpu malfunction. If the problem is
not "
-
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/, /, man/
commit: 75a4f2c2e07c128fef4d9faf3a8fb9d67565239e
Author: Sam James gentoo org>
AuthorDate: Thu Feb 16 06:26:07 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue Feb 21 07:16:27 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=75a4f2c2
emerge: add --update-if-installed
This adds a new emerge option '--update-if-installed'.
The use case for such an option is as follows:
- User finds out libfoo-1.2 is buggy.
- They want to upgrade all their systems if libfoo is installed.
- They don't want to install libfoo if it's not already installed.
Unfortunately, --update fails this last point, hence
the need for a new option.
Closes: https://github.com/gentoo/portage/pull/988
Signed-off-by: Sam James gentoo.org>
NEWS | 4 ++
lib/_emerge/create_depgraph_params.py | 6 ++
lib/_emerge/depgraph.py | 17 -
lib/_emerge/main.py | 3 +-
lib/portage/tests/resolver/test_update.py | 106 ++
man/emerge.1 | 6 ++
6 files changed, 140 insertions(+), 2 deletions(-)
diff --git a/NEWS b/NEWS
index 29e9de038..9389d2c09 100644
--- a/NEWS
+++ b/NEWS
@@ -10,6 +10,10 @@ Features:
* emerge: add --onlydeps-with-ideps= option (bug #890777)
+* emerge: add --update-if-installed option. This is useful for one-shot
+ emerge commands to be run across several machines to upgrade packages
+ only if they're installed.
+
* install-qa-check.d: 60pkgconfig: add opt-in QA_PKGCONFIG_VERSION check
* emerge: Log completion of package installs.
diff --git a/lib/_emerge/create_depgraph_params.py
b/lib/_emerge/create_depgraph_params.py
index 531230402..1bbca5de9 100644
--- a/lib/_emerge/create_depgraph_params.py
+++ b/lib/_emerge/create_depgraph_params.py
@@ -129,6 +129,12 @@ def create_depgraph_params(myopts, myaction):
if changed_slot:
myparams["changed_slot"] = True
+# --update-if-installed implies --update
+update_if_installed = myopts.get("--update-if-installed")
+if update_if_installed is not None:
+myparams["update_if_installed"] = update_if_installed
+myopts["--update"] = True
+
if (
"--update" in myopts
or "--newrepo" in myopts
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 1631ed126..412dc7b6f 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -5019,6 +5019,21 @@ class depgraph:
pkg, existing_node = self._select_package(
myroot, atom, onlydeps=onlydeps
)
+
+# Is the package installed (at any version)?
+if pkg and "update_if_installed" in
self._dynamic_config.myparams:
+package_is_installed = any(
+self._iter_match_pkgs(
+self._frozen_config.roots[myroot],
"installed", atom
+)
+)
+
+# This package isn't eligible for selection in the
+# merge list as the user passed --update-if-installed
+# and it isn't installed.
+if not package_is_installed:
+continue
+
if not pkg:
pprovided_match = False
for virt_choice in virtuals.get(atom.cp, []):
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 38233e05c..850487d36 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import argparse
@@ -52,6 +52,7 @@ options = [
"--tree",
"--unordered-display",
"--update",
+"--update-if-installed",
]
shortmapping = {
diff --git a/lib/portage/tests/resolver/test_update.py
b/lib/portage/tests/resolver/test_update.py
new file mode 100644
index 0..e67013f9f
--- /dev/null
+++ b/lib/portage/tests/resolver/test_update.py
@@ -0,0 +1,106 @@
+# Copyright 2022-2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+ResolverPlayground,
+ResolverPlaygroundTestCase,
+)
+
+
+class UpdateIfInstalledTestCase(TestCase):
+def testUpdateIfInstalledEmerge(self):
+installed = {
+"dev-lang/ghc-4": {},
+"dev-libs/larryware-3": {},
+"dev-libs/larryware-ng-3": {},
+"virtual/libc-1": {},
+}
+
+ebuilds = installed.copy()
+ebuilds.update(
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 0f075baba8e3accc124c9c599c90603e17cdf6a5
Author: Benjamin Gordon chromium org>
AuthorDate: Tue Feb 4 18:40:13 2020 +
Commit: Sam James gentoo org>
CommitDate: Sat Feb 18 00:00:12 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0f075bab
emerge: Factor out two functions
Will be used in the next commit.
Reviewed-by: Mike Frysinger chromium.org>
Reviewed-by: Chris McDonald chromium.org>
Signed-off-by: Benjamin Gordon chromium.org>
Signed-off-by: Matt Turner gentoo.org>
Signed-off-by: Sam James gentoo.org>
lib/_emerge/PackageMerge.py | 42 +-
1 file changed, 25 insertions(+), 17 deletions(-)
diff --git a/lib/_emerge/PackageMerge.py b/lib/_emerge/PackageMerge.py
index 8272511a1..f9e6bf8fe 100644
--- a/lib/_emerge/PackageMerge.py
+++ b/lib/_emerge/PackageMerge.py
@@ -9,13 +9,33 @@ from portage.output import colorize
class PackageMerge(CompositeTask):
__slots__ = ("merge", "postinst_failure")
+def _should_show_status(self):
+return (
+not self.merge.build_opts.fetchonly
+and not self.merge.build_opts.pretend
+and not self.merge.build_opts.buildpkgonly
+)
+
+def _make_msg(self, pkg, action_desc, preposition, counter_str):
+pkg_color = "PKG_MERGE"
+if pkg.type_name == "binary":
+pkg_color = "PKG_BINARY_MERGE"
+
+msg = "{} {}{}".format(
+action_desc,
+counter_str,
+colorize(pkg_color, pkg.cpv + _repo_separator + pkg.repo),
+)
+
+if pkg.root_config.settings["ROOT"] != "/":
+msg += f" {preposition} {pkg.root}"
+
+return msg
+
def _start(self):
self.scheduler = self.merge.scheduler
pkg = self.merge.pkg
pkg_count = self.merge.pkg_count
-pkg_color = "PKG_MERGE"
-if pkg.type_name == "binary":
-pkg_color = "PKG_BINARY_MERGE"
if pkg.installed:
action_desc = "Uninstalling"
@@ -29,20 +49,8 @@ class PackageMerge(CompositeTask):
colorize("MERGE_LIST_PROGRESS", str(pkg_count.maxval)),
)
-msg = "{} {}{}".format(
-action_desc,
-counter_str,
-colorize(pkg_color, pkg.cpv + _repo_separator + pkg.repo),
-)
-
-if pkg.root_config.settings["ROOT"] != "/":
-msg += f" {preposition} {pkg.root}"
-
-if (
-not self.merge.build_opts.fetchonly
-and not self.merge.build_opts.pretend
-and not self.merge.build_opts.buildpkgonly
-):
+if self._should_show_status():
+msg = self._make_msg(pkg, action_desc, preposition, counter_str)
self.merge.statusMessage(msg)
task = self.merge.create_install_task()
[gentoo-commits] proj/portage:master commit in: /, lib/_emerge/
commit: cdc8191e17e7dd1a6af1579ee2514a69811cf97b
Author: Benjamin Gordon chromium org>
AuthorDate: Tue Feb 4 18:40:13 2020 +
Commit: Sam James gentoo org>
CommitDate: Sat Feb 18 00:00:12 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cdc8191e
emerge: Log completion of package installs
The log currently shows Emerging/Installing progress messages, but
it doesn't indicate when a package finishes. Since there can be dozens
of installs in flight at once, this makes it nearly impossible to
determine how long an individual package took. This adds a similar
"Completed" line when the package merge completes.
Closes: https://github.com/gentoo/portage/pull/987
Reviewed-by: Mike Frysinger chromium.org>
Reviewed-by: Chris McDonald chromium.org>
Signed-off-by: Benjamin Gordon chromium.org>
Signed-off-by: Matt Turner gentoo.org>
Signed-off-by: Sam James gentoo.org>
NEWS| 2 ++
lib/_emerge/PackageMerge.py | 20
2 files changed, 22 insertions(+)
diff --git a/NEWS b/NEWS
index fbc847c37..8a4d95144 100644
--- a/NEWS
+++ b/NEWS
@@ -12,6 +12,8 @@ Features:
* install-qa-check.d: 60pkgconfig: add opt-in QA_PKGCONFIG_VERSION check
+* emerge: Log completion of package installs.
+
Bug fixes:
* gpkg: Handle out-of-space errors (bug #891391).
diff --git a/lib/_emerge/PackageMerge.py b/lib/_emerge/PackageMerge.py
index f9e6bf8fe..82725c66a 100644
--- a/lib/_emerge/PackageMerge.py
+++ b/lib/_emerge/PackageMerge.py
@@ -58,5 +58,25 @@ class PackageMerge(CompositeTask):
def _install_exit(self, task):
self.postinst_failure = getattr(task, "postinst_failure", None)
+
+pkg = self.merge.pkg
+pkg_count = self.merge.pkg_count
+
+if self.postinst_failure:
+action_desc = "Failed"
+preposition = "in"
+counter_str = ""
+else:
+action_desc = "Completed"
+preposition = "to"
+counter_str = "({} of {}) ".format(
+colorize("MERGE_LIST_PROGRESS", str(pkg_count.curval)),
+colorize("MERGE_LIST_PROGRESS", str(pkg_count.maxval)),
+)
+
+if self._should_show_status():
+msg = self._make_msg(pkg, action_desc, preposition, counter_str)
+self.merge.statusMessage(msg)
+
self._final_exit(task)
self.wait()
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 8bda7308b3f481ea05f0b0c671dfec4024384dfa
Author: Sheng Yu protonmail com>
AuthorDate: Sat Feb 11 03:10:23 2023 +
Commit: Sam James gentoo org>
CommitDate: Fri Feb 17 01:23:14 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8bda7308
Add warning if missing PATH in binpkg index
Outdated portage does not have this metadata by default.
Bug: https://bugs.gentoo.org/884869
Signed-off-by: Sheng Yu protonmail.com>
Signed-off-by: Sam James gentoo.org>
lib/_emerge/BinpkgFetcher.py | 4
1 file changed, 4 insertions(+)
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index df99486a0..a18bed759 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -28,6 +28,10 @@ class BinpkgFetcher(CompositeTask):
instance_key = bintree.dbapi._instance_key(pkg.cpv)
binpkg_path = bintree._remotepkgs[instance_key].get("PATH")
+if not binpkg_path:
+raise FileNotFound(
+f"PATH not found in the binpkg index, the binhost's portage is
probably out of date."
+)
binpkg_format = get_binpkg_format(binpkg_path)
self.pkg_allocated_path = pkg.root_config.trees["bintree"].getname(
[gentoo-commits] proj/portage:master commit in: lib/_emerge/resolver/, lib/_emerge/, lib/portage/dep/
commit: 32d6700c84d4c41392f26eab89ffb48c6a4104fd
Author: Oskari Pirhonen gmail com>
AuthorDate: Sat Feb 11 02:14:10 2023 +
Commit: Sam James gentoo org>
CommitDate: Sat Feb 11 02:54:54 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=32d6700c
Fix issues causing CI to fail
Signed-off-by: Oskari Pirhonen gmail.com>
Closes: https://github.com/gentoo/portage/pull/985
Signed-off-by: Sam James gentoo.org>
lib/_emerge/AbstractEbuildProcess.py | 4 +++-
lib/_emerge/resolver/backtracking.py | 2 +-
lib/portage/dep/__init__.py | 4 ++--
3 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/lib/_emerge/AbstractEbuildProcess.py
b/lib/_emerge/AbstractEbuildProcess.py
index f457e4609..0e6786eed 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -27,7 +27,9 @@ portage.proxy.lazyimport.lazyimport(
class AbstractEbuildProcess(SpawnProcess):
-__slots__ = ("phase", "settings",) + (
+__slots__ = (
+"phase",
+"settings",
"_build_dir",
"_build_dir_unlock",
"_ipc_daemon",
diff --git a/lib/_emerge/resolver/backtracking.py
b/lib/_emerge/resolver/backtracking.py
index 6fbfdd89c..9cbe65411 100644
--- a/lib/_emerge/resolver/backtracking.py
+++ b/lib/_emerge/resolver/backtracking.py
@@ -270,7 +270,7 @@ class Backtracker:
# Not all config changes require a restart, that's why they can appear
together
# with other conflicts.
if "config" in infos:
-self._feedback_config(infos["config"], explore=(len(infos) == 1))
+self._feedback_config(infos["config"], explore=len(infos) == 1)
# There is at most one of the following types of conflicts for a given
restart.
if "slot conflict" in infos:
diff --git a/lib/portage/dep/__init__.py b/lib/portage/dep/__init__.py
index 889d98263..d0c5a45cc 100644
--- a/lib/portage/dep/__init__.py
+++ b/lib/portage/dep/__init__.py
@@ -1499,8 +1499,8 @@ class Atom(str):
allow_build_id = True
blocker_prefix = ""
-if "!" == s[:1]:
-blocker = self._blocker(forbid_overlap=("!" == s[1:2]))
+if s[:1] == "!":
+blocker = self._blocker(forbid_overlap=s[1:2] == "!")
if blocker.overlap.forbid:
blocker_prefix = s[:2]
s = s[2:]
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, man/, cnf/
commit: 7dfa05c4446d1d2290d4ba61520a8f6ee9e57332
Author: KARBOWSKI Piotr gentoo org>
AuthorDate: Sat Jan 14 20:13:40 2023 +
Commit: Mike Gilbert gentoo org>
CommitDate: Sat Jan 14 21:02:21 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7dfa05c4
SCHEDULING_POLICY: switch to scheduler policy IDs rather than aliases.
The os.SCHED_* aliases are not reliable enough, the mainline Python lacks
deadline, and the pypy lacks them all together. The IDs are not going to
change, at most new would arrive, so it is safe to use it as is.
By extension, the deadline policy was also added, since those originally
were not present in aliases.
Bug: https://bugs.gentoo.org/867031
Closes: https://github.com/gentoo/portage/pull/976
Signed-off-by: KARBOWSKI Piotr gentoo.org>
Signed-off-by: Mike Gilbert gentoo.org>
cnf/make.conf.example | 12 ++--
lib/_emerge/actions.py | 12 +++-
man/make.conf.5| 16
3 files changed, 21 insertions(+), 19 deletions(-)
diff --git a/cnf/make.conf.example b/cnf/make.conf.example
index 2e33a6e50..4375665f4 100644
--- a/cnf/make.conf.example
+++ b/cnf/make.conf.example
@@ -292,12 +292,12 @@
#PORTAGE_IONICE_COMMAND="ionice -c 3 -p \${PID}"
#
# PORTAGE_SCHEDULING_POLICY allows changing the current scheduling policy. The
-# supported options are 'other', 'batch', 'idle', 'fifo' and 'round-robin'.
When
-# unset, the scheduling policy remains unchanged, by default Linux uses 'other'
-# policy. Users that wish to minimize the Portage's impact on system
-# responsiveness should set scheduling policy to 'idle' which significantly
-# reduces the disruption to the rest of the system by scheduling Portage as
-# extremely low priority processes.
+# supported options are 'other', 'batch', 'idle', 'fifo', 'round-robin' and
+# 'deadline'. When unset, the scheduling policy remains unchanged, by default
+# Linux uses 'other' policy. Users that wish to minimize the Portage's impact
+# on system responsiveness should set scheduling policy to 'idle' which
+# significantly reduces the disruption to the rest of the system by scheduling
+# Portage as extremely low priority processes.
#
#PORTAGE_SCHEDULING_POLICY="idle"
#
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 874c0312c..d8fd99d88 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -3117,12 +3117,14 @@ def set_scheduling_policy(settings):
if platform.system() != "Linux" or not scheduling_policy:
return os.EX_OK
+# IDs sourced from linux/sched.h kernel's header.
policies = {
-"other": os.SCHED_OTHER,
-"batch": os.SCHED_BATCH,
-"idle": os.SCHED_IDLE,
-"fifo": os.SCHED_FIFO,
-"round-robin": os.SCHED_RR,
+"other": 0,
+"fifo": 1,
+"round-robin": 2,
+"batch": 3,
+"idle": 5,
+"deadline": 6,
}
out = portage.output.EOutput()
diff --git a/man/make.conf.5 b/man/make.conf.5
index ad8ece43f..34c3cd5b5 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -1101,14 +1101,14 @@ will set idle io priority. For more information about
ionice, see
Portage will also set the autogroup-nice value (see fBsched\fR(7))), if
FEATURES="pid\-sandbox" is enabled.
.TP
-\fBPORTAGE_SCHEDULING_POLICY\fR = \fI[policy name]\fR
-Allows changing the current scheduling policy. The supported options are
-\fBother\fR, \fBbatch\fR, \fBidle\fR, \fBfifo\fR, and \fBround-robin\fR. When
-unset, the scheduling policy remains unchanged, by default Linux uses 'other'
-policy. Users that wish to minimize the Portage's impact on system
-responsiveness should set scheduling policy to \fBidle\fR, which significantly
-reduces the disruption to the rest of the system by scheduling Portage as
-extremely low priority processes. see \fBsched\fR(7) for more information.
+\fBPORTAGE_SCHEDULING_POLICY\fR = \fI[policy name]\fR Allows changing the
+current scheduling policy. The supported options are \fBother\fR, \fBbatch\fR,
+\fBidle\fR, \fBfifo\fR, \fBround-robin\fR and \fBdeadline\fR. When unset, the
+scheduling policy remains unchanged, by default Linux uses 'other' policy.
+Users that wish to minimize the Portage's impact on system responsiveness
+should set scheduling policy to \fBidle\fR, which significantly reduces the
+disruption to the rest of the system by scheduling Portage as extremely low
+priority processes. see \fBsched\fR(7) for more information.
.TP
\fBPORTAGE_SCHEDULING_PRIORITY\fR = \fI[priority]\fR
Allows changing the priority (1-99) of the current scheduling policy, only
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 5b8ae94c1bf5f9d3bd7e7970f9cdb0eb1f11a078
Author: Sam James gentoo org>
AuthorDate: Tue Jan 10 13:28:45 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue Jan 10 15:12:39 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5b8ae94c
*/*: run Black again (post-pyupgrade)
Signed-off-by: Sam James gentoo.org>
lib/_emerge/actions.py | 4 +---
lib/_emerge/depgraph.py | 4 +---
2 files changed, 2 insertions(+), 6 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index ba6d99d25..874c0312c 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1643,9 +1643,7 @@ def _calc_depclean(settings, trees, ldpath_mtimes,
myopts, action, args_set, spi
if debug:
writemsg_level(
-"Candidates: [{}]\n".format(
-", ".join(f"'{x}'" for x in atoms)
-),
+"Candidates: [{}]\n".format(", ".join(f"'{x}'" for x
in atoms)),
noiselevel=-1,
level=logging.DEBUG,
)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 76d0950ae..a88c7a1fa 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -2037,9 +2037,7 @@ class depgraph:
f" package(s) to mask: {str(to_be_masked)}",
f" slot: {slot_atom}",
" parents: {}".format(
-", ".join(
-f"({ppkg}, '{atom}')" for ppkg, atom in all_parents
-)
+", ".join(f"({ppkg}, '{atom}')" for ppkg, atom in
all_parents)
),
"",
]
[gentoo-commits] proj/portage:master commit in: /, lib/_emerge/
commit: a0ca453907f461d0bed987cfe7ce52520b2d59ef
Author: Sheng Yu protonmail com>
AuthorDate: Tue Jan 10 03:03:40 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue Jan 10 03:28:03 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a0ca4539
emerge: depgraph: Correctly detect installation from path for binpkgs
Bug: https://bugs.gentoo.org/873202
Bug: https://bugs.gentoo.org/890291
Signed-off-by: Sheng Yu protonmail.com>
Closes: https://github.com/gentoo/portage/pull/973
Signed-off-by: Sam James gentoo.org>
NEWS| 5 -
lib/_emerge/depgraph.py | 14 +++---
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/NEWS b/NEWS
index 3821e6fcf..d143765b1 100644
--- a/NEWS
+++ b/NEWS
@@ -1,10 +1,13 @@
-portage-3.0.44 (?)
+portage-3.0.44 (UNRELEASED)
+--
Bug fixes:
* ebuild: the PATH variable exported to ebuilds has been changed:
The PATH setting from /etc/profile.env is appended to portage-internal
paths, and ROOTPATH is no longer included (bug #607696, #693308, #888543).
+* emerge: Fix installation of binpkgs by path (bug #873202, bug #890291).
+
portage-3.0.43 (2023-01-02)
--
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 6b08f9fe2..4d7a96a2d 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -18,6 +18,7 @@ from portage.const import (
PORTAGE_PACKAGE_ATOM,
USER_CONFIG_PATH,
VCS_DIRS,
+SUPPORTED_XPAK_EXTENSIONS,
SUPPORTED_GPKG_EXTENSIONS,
)
from portage.dbapi import dbapi
@@ -4574,7 +4575,9 @@ class depgraph:
onlydeps = "--onlydeps" in self._frozen_config.myopts
lookup_owners = []
for x in myfiles:
-if x.endswith(".tbz2") or x.endswith(SUPPORTED_GPKG_EXTENSIONS):
+if x.endswith(SUPPORTED_XPAK_EXTENSIONS) or x.endswith(
+SUPPORTED_GPKG_EXTENSIONS
+):
if not os.path.exists(x):
if os.path.exists(os.path.join(pkgsettings["PKGDIR"],
"All", x)):
x = os.path.join(pkgsettings["PKGDIR"], "All", x)
@@ -4608,7 +4611,7 @@ class depgraph:
mykey = None
cat = mytbz2.getfile("CATEGORY")
elif binpkg_format == "gpkg":
-mygpkg = portage.gpkg.gpkg(self.frozen_config, None, x)
+mygpkg = portage.gpkg.gpkg(root_config.settings, None, x)
mykey = None
cat = mygpkg.get_metadata("CATEGORY")
else:
@@ -4618,7 +4621,12 @@ class depgraph:
cat = _unicode_decode(
cat.strip(), encoding=_encodings["repo.content"]
)
-mykey = cat + "/" + os.path.basename(x)[:-5]
+if binpkg_format == "xpak":
+mykey = cat + "/" + os.path.basename(x)[:-5]
+elif binpkg_format == "gpkg":
+mykey = cat + "/" + os.path.basename(x)[:-9]
+else:
+raise InvalidBinaryPackageFormat(x)
if mykey is None:
writemsg(
[gentoo-commits] proj/portage:master commit in: /, lib/_emerge/
commit: 7b47c30537c58c6c9a25b18959c58de85067634d
Author: Michał Górny gentoo org>
AuthorDate: Thu Jan 5 15:26:41 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue Jan 10 03:28:46 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7b47c305
depgraph: Print dependency resolution time
Print the wall clock time it took emerge to calculate the dependency
graph if spinner is used (i.e. we're in verbose mode).
Requested-by: Maciej Barć gentoo.org>
Signed-off-by: Michał Górny gentoo.org>
Closes: https://github.com/gentoo/portage/pull/972
Signed-off-by: Sam James gentoo.org>
NEWS | 4
lib/_emerge/depgraph.py | 6 ++
lib/_emerge/stdout_spinner.py | 1 +
3 files changed, 11 insertions(+)
diff --git a/NEWS b/NEWS
index d143765b1..23a652278 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,10 @@
portage-3.0.44 (UNRELEASED)
--
+Features:
+* emerge: Show time taken to calculate dependency resolution with
+ emerge --verbose.
+
Bug fixes:
* ebuild: the PATH variable exported to ebuilds has been changed:
The PATH setting from /etc/profile.env is appended to portage-internal
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 4d7a96a2d..15dcfbc84 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -6,6 +6,7 @@ import functools
import logging
import stat
import textwrap
+import time
import warnings
import collections
from collections import deque, OrderedDict
@@ -11522,6 +11523,7 @@ def _spinner_start(spinner, myopts):
if show_spinner:
portage.writemsg_stdout("Calculating dependencies ")
+spinner.start_time = time.time()
def _spinner_stop(spinner):
@@ -11535,6 +11537,10 @@ def _spinner_stop(spinner):
portage.writemsg_stdout("... done!\n")
+stop_time = time.time()
+time_fmt = f"{stop_time - spinner.start_time:.2f}"
+portage.writemsg_stdout(f"Dependency resolution took {darkgreen(time_fmt)}
s.\n\n")
+
def backtrack_depgraph(settings, trees, myopts, myparams, myaction, myfiles,
spinner):
"""
diff --git a/lib/_emerge/stdout_spinner.py b/lib/_emerge/stdout_spinner.py
index b28977e66..d4c1db59b 100644
--- a/lib/_emerge/stdout_spinner.py
+++ b/lib/_emerge/stdout_spinner.py
@@ -39,6 +39,7 @@ class stdout_spinner:
]
self.last_update = 0
self.min_display_latency = 0.05
+self.start_time = None
def _return_early(self):
"""
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, /, lib/portage/elog/
commit: a28764138198e641cd0fdcc166d596f2cd076225
Author: James Le Cuirot gentoo org>
AuthorDate: Tue Dec 13 21:28:22 2022 +
Commit: Sam James gentoo org>
CommitDate: Thu Dec 15 00:50:37 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a2876413
logging: Write logs under BROOT, not EPREFIX
Logs should be written to the build system, not the host system. When
building for a prefixed system under ROOT, logs were being written under
/${EPREFIX} even though that directory did not exist at all.
Signed-off-by: James Le Cuirot gentoo.org>
Closes: https://github.com/gentoo/portage/pull/959
Signed-off-by: Sam James gentoo.org>
NEWS | 2 ++
lib/_emerge/actions.py | 5 -
lib/portage/elog/mod_save.py | 2 +-
lib/portage/elog/mod_save_summary.py | 2 +-
4 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/NEWS b/NEWS
index 4684487b3..e80c82227 100644
--- a/NEWS
+++ b/NEWS
@@ -15,6 +15,8 @@ Bug fixes:
* make.conf(5): Note the various file suffixes / file extensions used
for binary packages.
+* logging: Write logs under BROOT, not EPREFIX.
+
portage-3.0.41 (2022-11-04)
--
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 30b687140..047948f56 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -3799,7 +3799,10 @@ def run_action(emerge_config):
emerge_log_dir = emerge_config.target_config.settings.get("EMERGE_LOG_DIR")
default_log_dir = os.path.join(
-os.sep, portage.const.EPREFIX.lstrip(os.sep), "var", "log"
+os.sep,
+emerge_config.target_config.settings["BROOT"].lstrip(os.sep),
+"var",
+"log",
)
for x in ("--pretend", "--fetchonly", "--fetch-all-uri"):
if x in emerge_config.opts:
diff --git a/lib/portage/elog/mod_save.py b/lib/portage/elog/mod_save.py
index a2b2cf7bd..e526b331f 100644
--- a/lib/portage/elog/mod_save.py
+++ b/lib/portage/elog/mod_save.py
@@ -20,7 +20,7 @@ def process(mysettings, key, logentries, fulltext):
logdir = normalize_path(mysettings["PORTAGE_LOGDIR"])
else:
logdir = os.path.join(
-os.sep, mysettings["EPREFIX"].lstrip(os.sep), "var", "log",
"portage"
+os.sep, mysettings["BROOT"].lstrip(os.sep), "var", "log", "portage"
)
if not os.path.isdir(logdir):
diff --git a/lib/portage/elog/mod_save_summary.py
b/lib/portage/elog/mod_save_summary.py
index 5aa31a8e1..f2913da5b 100644
--- a/lib/portage/elog/mod_save_summary.py
+++ b/lib/portage/elog/mod_save_summary.py
@@ -20,7 +20,7 @@ def process(mysettings, key, logentries, fulltext):
logdir = normalize_path(mysettings["PORTAGE_LOGDIR"])
else:
logdir = os.path.join(
-os.sep, mysettings["EPREFIX"].lstrip(os.sep), "var", "log",
"portage"
+os.sep, mysettings["BROOT"].lstrip(os.sep), "var", "log", "portage"
)
if not os.path.isdir(logdir):
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 0c42cc962e1926ecbdc83d903a2804f9e037f2a9
Author: Pin-yen Lin chromium org>
AuthorDate: Fri Nov 25 03:36:26 2022 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Nov 28 00:37:41 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0c42cc96
Move and rename check_reverse_dependencies
Move the function outside the original scope and rename the function to
_slot_operator_check_reverse_dependencies to accommodate the coding
style.
Signed-off-by: Pin-yen Lin chromium.org>
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/depgraph.py | 226
1 file changed, 113 insertions(+), 113 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index e556d6616..bb0824324 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -2240,6 +2240,111 @@ class depgraph:
return None
+def _slot_operator_check_reverse_dependencies(
+self, existing_pkg, candidate_pkg, replacement_parent=None
+):
+"""
+Check if candidate_pkg satisfies all of existing_pkg's non-
+slot operator parents.
+"""
+debug = "--debug" in self._frozen_config.myopts
+built_slot_operator_parents = set()
+for parent, atom in
self._dynamic_config._parent_atoms.get(existing_pkg, []):
+if atom.soname or atom.slot_operator_built:
+built_slot_operator_parents.add(parent)
+
+for parent, atom in
self._dynamic_config._parent_atoms.get(existing_pkg, []):
+if isinstance(parent, Package):
+if parent in built_slot_operator_parents:
+if hasattr(atom, "_orig_atom"):
+# If atom is the result of virtual expansion, then
+# derefrence it to _orig_atom so that it will be
correctly
+# handled as a built slot operator dependency when
+# appropriate (see bug 764764).
+atom = atom._orig_atom
+# This parent may need to be rebuilt, therefore
+# discard its soname and built slot operator
+# dependency components which are not necessarily
+# relevant.
+if atom.soname:
+continue
+elif atom.package and atom.slot_operator_built:
+# This discards the slot/subslot component.
+atom = atom.with_slot("=")
+
+if replacement_parent is not None and (
+replacement_parent.slot_atom == parent.slot_atom
+or replacement_parent.cpv == parent.cpv
+):
+# This parent is irrelevant because we intend to
+# replace it with replacement_parent.
+continue
+
+if any(
+pkg is not parent
+and (pkg.slot_atom == parent.slot_atom or pkg.cpv ==
parent.cpv)
+for pkg in self._dynamic_config._package_tracker.match(
+parent.root, Atom(parent.cp)
+)
+):
+# This parent may need to be eliminated due to a
+# slot conflict, so its dependencies aren't
+# necessarily relevant.
+continue
+
+if not self._too_deep(
+parent.depth
+) and not self._frozen_config.excluded_pkgs.findAtomForPackage(
+parent, modified_use=self._pkg_use_enabled(parent)
+):
+# Check for common reasons that the parent's
+# dependency might be irrelevant.
+if self._upgrade_available(parent):
+# This parent could be replaced by
+# an upgrade (bug 584626).
+continue
+if parent.installed and self._in_blocker_conflict(parent):
+# This parent could be uninstalled in order
+# to solve a blocker conflict (bug 612772).
+continue
+if self._dynamic_config.digraph.has_edge(parent,
existing_pkg):
+# There is a direct circular dependency between
+# parent and existing_pkg. This type of
+# relationship tends to prevent updates
+# of packages (bug 612874). Since candidate_pkg
+# is available, we risk a missed update if we
+# don't try to eliminate this parent from the
+# graph. Therefore, we give candidate_pkg a
+# chance, and assume that it will be masked
+# by backtracking if
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 839ab46be1777e5886da28b98b53a462b992c5bf Author: Pin-yen Lin chromium org> AuthorDate: Fri Nov 25 03:36:27 2022 + Commit: Zac Medico gentoo org> CommitDate: Mon Nov 28 00:37:41 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=839ab46b Add caching to _slot_operator_check_reverse_dependencies Add lru_cache to speed up the running time of "Calculating dependencies". In a ChromeOS use case, this patch decreases the running time from 311s to 197s with almost no memory usage increase. Bug: https://bugs.gentoo.org/883071 Signed-off-by: Pin-yen Lin chromium.org> Signed-off-by: Zac Medico gentoo.org> lib/_emerge/depgraph.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index bb0824324..d0ea92ad9 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -2240,6 +2240,7 @@ class depgraph: return None +@functools.lru_cache(maxsize=100) def _slot_operator_check_reverse_dependencies( self, existing_pkg, candidate_pkg, replacement_parent=None ):
[gentoo-commits] proj/portage:master commit in: /, lib/_emerge/
commit: fca3478de7127bbdf58c6f4b809c177662d5e4cb Author: James Le Cuirot gentoo org> AuthorDate: Sun Nov 20 11:46:34 2022 + Commit: Sam James gentoo org> CommitDate: Wed Nov 23 23:01:57 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fca3478d depgraph: Resolve atoms using the correct root This bug may have played out in different ways, but one example was a rebuild in / causing the same package to be added to ROOT, even when it had no other reason to be. Signed-off-by: James Le Cuirot gentoo.org> Closes: https://github.com/gentoo/portage/pull/941 Signed-off-by: Sam James gentoo.org> NEWS| 4 lib/_emerge/depgraph.py | 14 ++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 1426a0856..8767d393c 100644 --- a/NEWS +++ b/NEWS @@ -31,6 +31,10 @@ Bug fixes: by the SYSROOT location. For example, if BROOT=/foo, ROOT=/bar, EPREFIX=/baz, EROOT=/bar/baz, and SYSROOT=/ then ESYSROOT should be /foo, not /baz. +* depgraph: Resolve atoms using the correct root. This bug may have played out + in different ways, but one example was a rebuild in / causing the same package + to be added to ROOT, even when it had no other reason to be. + portage-3.0.39 (2022-11-20) -- diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index c976f8205..e556d6616 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -5019,13 +5019,14 @@ class depgraph: a favorite list.""" debug = "--debug" in self._frozen_config.myopts onlydeps = "--onlydeps" in self._frozen_config.myopts -myroot = self._frozen_config.target_root -pkgsettings = self._frozen_config.pkgsettings[myroot] -pprovideddict = pkgsettings.pprovideddict -virtuals = pkgsettings.getvirtuals() args = self._dynamic_config._initial_arg_list[:] for arg in self._expand_set_args(args, add_to_digraph=True): +myroot = arg.root_config.root +pkgsettings = self._frozen_config.pkgsettings[myroot] +pprovideddict = pkgsettings.pprovideddict +virtuals = pkgsettings.getvirtuals() + for atom in sorted(arg.pset.getAtoms()): self._spinner_update() dep = Dependency(atom=atom, onlydeps=onlydeps, root=myroot, parent=arg) @@ -5169,6 +5170,11 @@ class depgraph: ) raise +try: +del myroot, pkgsettings, pprovideddict, virtuals +except NameError: +pass + # Now that the root packages have been added to the graph, # process the dependencies. if not self._create_graph():
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, /, lib/portage/util/, lib/portage/, bin/, ...
commit: d0fa3d28c5de6a0a34ac87de5e1a463adbe58405
Author: Sam James gentoo org>
AuthorDate: Thu Sep 29 06:37:19 2022 +
Commit: Sam James gentoo org>
CommitDate: Tue Oct 11 19:18:35 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d0fa3d28
binpkg: compress/decompress xz in parallel; compress zstd in parallel
- As before, needs >= xz-5.3.3_alpha for parallel decompression.
- zstd does not support parallel decompression.
Closes: https://github.com/gentoo/portage/pull/918
Signed-off-by: Sam James gentoo.org>
NEWS | 7 +++
bin/quickpkg | 8 +++-
lib/_emerge/BinpkgExtractorAsync.py| 4
lib/portage/gpkg.py| 8 +++-
lib/portage/package/ebuild/doebuild.py | 14 +++---
lib/portage/util/compression_probe.py | 6 +++---
lib/portage/util/cpuinfo.py| 22 +-
7 files changed, 60 insertions(+), 9 deletions(-)
diff --git a/NEWS b/NEWS
index 28e8f97ca..a3591e94d 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,10 @@
+portage-3.0.39 (UNRELEASED)
+--
+
+Features:
+* gpkg: Call xz and zstd with -T N where N is the number of jobs from
${MAKEOPTS}
+ for parallel compression (xz & zstd) and decompression (xz with newer
versions).
+
portage-3.0.38.1 (2022-10-04)
--
diff --git a/bin/quickpkg b/bin/quickpkg
index 67eee697a..9ff8d7b5a 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -36,6 +36,7 @@ from portage.exception import (
PermissionDenied,
)
from portage.util import ensure_dirs, shlex_split, varexpand, _xattr
+from portage.util.cpuinfo import makeopts_to_job_count
xattr = _xattr.xattr
from portage._sets import load_default_config, SETPREFIX
@@ -168,7 +169,12 @@ def quickpkg_atom(options, infos, arg, eout):
% (binpkg_compression, missing_package)
)
return 1
-cmd = shlex_split(varexpand(compression["compress"],
mydict=settings))
+
+cmd = compression["compress"].replace(
+"{JOBS}",
+str(makeopts_to_job_count(settings.get("MAKEOPTS", "1"))),
+)
+cmd = shlex_split(varexpand(cmd, mydict=settings))
# Filter empty elements that make Popen fail
cmd = [x for x in cmd if x != ""]
with open(binpkg_tmpfile, "wb") as fobj:
diff --git a/lib/_emerge/BinpkgExtractorAsync.py
b/lib/_emerge/BinpkgExtractorAsync.py
index 919837fc1..65b383998 100644
--- a/lib/_emerge/BinpkgExtractorAsync.py
+++ b/lib/_emerge/BinpkgExtractorAsync.py
@@ -10,6 +10,7 @@ from portage.util.compression_probe import (
compression_probe,
_compressors,
)
+from portage.util.cpuinfo import makeopts_to_job_count
from portage.process import find_binary
from portage.util import (
shlex_split,
@@ -53,6 +54,9 @@ class BinpkgExtractorAsync(SpawnProcess):
decomp = _compressors.get(compression_probe(self.pkg_path))
if decomp is not None:
decomp_cmd = decomp.get("decompress")
+decomp_cmd = decomp_cmd.replace(
+"{JOBS}", str(makeopts_to_job_count(self.env.get("MAKEOPTS",
"1")))
+)
elif tarfile.is_tarfile(
portage._unicode_encode(
self.pkg_path, encoding=portage._encodings["fs"],
errors="strict"
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 7e1b98370..1f8fa1fd3 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -38,6 +38,7 @@ from portage.util._urlopen import urlopen
from portage.util import writemsg
from portage.util import shlex_split, varexpand
from portage.util.compression_probe import _compressors
+from portage.util.cpuinfo import makeopts_to_job_count
from portage.process import find_binary
from portage.const import MANIFEST2_HASH_DEFAULTS, HASHING_BLOCKSIZE
@@ -1787,7 +1788,12 @@ class gpkg:
if mode not in compressor:
raise InvalidCompressionMethod("{}: {}".format(compression, mode))
-cmd = shlex_split(varexpand(compressor[mode], mydict=self.settings))
+cmd = compressor[mode]
+cmd = cmd.replace(
+"{JOBS}", str(makeopts_to_job_count(self.settings.get("MAKEOPTS",
"1")))
+)
+cmd = shlex_split(varexpand(cmd, mydict=self.settings))
+
# Filter empty elements that make Popen fail
cmd = [x for x in cmd if x != ""]
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index d0d134b39..4f7049c21 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -111,7 +111,7 @@ from portage.util import (
writemsg_stdout,
write_atomic,
)
-from portage.util.cpuinfo import get_cpu_count
+from portage.util.cpuinfo import get_cpu_count, makeopts_to_job_count
from portage.util.lafilefixer import
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 6313f4e79868cf8e5aaf49751ab5dd0d5f776761 Author: David Palao gmail com> AuthorDate: Fri Sep 23 13:47:17 2022 + Commit: Mike Gilbert gentoo org> CommitDate: Sun Sep 25 19:11:55 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6313f4e7 improvement(style): black issue fixed Closes: https://github.com/gentoo/portage/pull/897 Signed-off-by: David Palao gmail.com> Signed-off-by: Mike Gilbert gentoo.org> lib/_emerge/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index fbfc561ec..b45a6361a 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -3512,7 +3512,7 @@ def run_action(emerge_config): mytrees["bintree"].populate( getbinpkgs="--getbinpkg" in emerge_config.opts, getbinpkg_refresh=True, -**kwargs +**kwargs, ) except ParseError as e: writemsg(
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/emerge/
commit: 7fe37c7bbf6c42fb0b5ea5d8a431abf677df5822
Author: David Palao gmail com>
AuthorDate: Fri Sep 16 13:48:39 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Sun Sep 25 19:10:01 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7fe37c7b
test(actions): requiring that run_action calls binarytree.populate correctly
...which means that since this is the first time, populate uses
getbinpkg_refresh=True explicitly.
This step is a preparation for a follow-up little change in the API of
binarytree.populate
Bug: https://bugs.gentoo.org/864259
Signed-off-by: David Palao gmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
lib/_emerge/actions.py | 4 ++-
lib/portage/tests/emerge/test_actions.py | 45
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 26120ad6d..fbfc561ec 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -3510,7 +3510,9 @@ def run_action(emerge_config):
try:
mytrees["bintree"].populate(
-getbinpkgs="--getbinpkg" in emerge_config.opts, **kwargs
+getbinpkgs="--getbinpkg" in emerge_config.opts,
+getbinpkg_refresh=True,
+**kwargs
)
except ParseError as e:
writemsg(
diff --git a/lib/portage/tests/emerge/test_actions.py
b/lib/portage/tests/emerge/test_actions.py
new file mode 100644
index 0..014d39dd9
--- /dev/null
+++ b/lib/portage/tests/emerge/test_actions.py
@@ -0,0 +1,45 @@
+# Copyright 2022 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from unittest.mock import MagicMock, patch
+
+from _emerge.actions import run_action
+from portage.tests import TestCase
+
+
+class RunActionTestCase(TestCase):
+"""This class' purpose is to encompass UTs for ``actions.run_action``.
+Since that function is extremely long (at least on Sep. 2022;
+hopefully the situation gets better with the time), the tests in this
+``TestCase`` contain plenty of mocks/patches.
+Hopefully, with time and effort, the ``run_action`` function (and others
+in the module) are refactored to make testing easier and more robust.
+
+A side effect of the mocking approach is a strong dependency on the
+details of the implementation. That can be improved if functions
+are smaller and do a well defined small set of tasks. Another call to
+refactoring...
+If the implementation changes, the mocks can be adjusted to play its
+role.
+"""
+@patch("_emerge.actions.profile_check")
+@patch("_emerge.actions.adjust_configs")
+@patch("_emerge.actions.apply_priorities")
+def test_binary_trees_populate_called(
+self,
+papply,
+padjust,
+profile_ckeck):
+config = MagicMock()
+config.action = None
+config.opts = {"--quiet": True, "--usepkg": True}
+bt = MagicMock()
+tree = {"bintree": bt}
+trees = {"first": tree}
+config.trees = trees
+
+run_action(config)
+
+bt.populate.assert_called_once_with(
+getbinpkgs=False, getbinpkg_refresh=True
+)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 7dc66f4f5635ce4abb97b3c6698cda3396cce820 Author: Sheng Yu protonmail com> AuthorDate: Fri Sep 23 20:32:01 2022 + Commit: Sam James gentoo org> CommitDate: Sun Sep 25 01:36:11 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7dc66f4f _emerge: Scheduler: Use pkg_allocated_path in Scheduler We previously forgot to plumb in allocated_pkg_path to use pkg_allocated_path which meant that Scheduler allocated a new one rather than the existing one. Bug: https://bugs.gentoo.org/872392 Signed-off-by: Sheng Yu protonmail.com> Closes: https://github.com/gentoo/portage/pull/906 Signed-off-by: Sam James gentoo.org> lib/_emerge/Scheduler.py | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index 9e210f182..356d6ce1f 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -975,7 +975,11 @@ class Scheduler(PollScheduler): continue if fetched: -bintree.inject(x.cpv, current_pkg_path=fetched) +bintree.inject( +x.cpv, +current_pkg_path=fetched, +allocated_pkg_path=fetcher.pkg_allocated_path, +) infloc = os.path.join(build_dir_path, "build-info") ensure_dirs(infloc)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, bin/
commit: 57ec5fc566a8e47fefad628fc61aa9a575341ff2
Author: Sheng Yu protonmail com>
AuthorDate: Wed Sep 14 10:01:02 2022 +
Commit: Sam James gentoo org>
CommitDate: Tue Sep 20 03:39:23 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=57ec5fc5
gpkg-sign: add gpg configuration check
Bug: https://bugs.gentoo.org/869470
Signed-off-by: Sheng Yu protonmail.com>
Closes: https://github.com/gentoo/portage/pull/899
Signed-off-by: Sam James gentoo.org>
bin/gpkg-sign | 9 +
lib/_emerge/actions.py | 16
2 files changed, 25 insertions(+)
diff --git a/bin/gpkg-sign b/bin/gpkg-sign
index 57fc6ce98..0aa6483e7 100755
--- a/bin/gpkg-sign
+++ b/bin/gpkg-sign
@@ -16,6 +16,15 @@ def main(
gpkg_file, keep_current_signature=False, allow_unsigned=False,
skip_signed=False
):
eout = portage.output.EOutput()
+
+if not portage.settings.get("BINPKG_GPG_SIGNING_GPG_HOME"):
+eout.eerror("BINPKG_GPG_SIGNING_GPG_HOME is not set")
+exit(1)
+
+if not portage.settings.get("BINPKG_GPG_SIGNING_KEY"):
+eout.eerror("BINPKG_GPG_SIGNING_KEY is not set")
+exit(1)
+
try:
package = gpkg.gpkg(settings=portage.settings, gpkg_file=gpkg_file)
if allow_unsigned:
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index e79bb30c0..26120ad6d 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -614,6 +614,22 @@ def action_build(
in trees[eroot]["root_config"].settings.features
)
):
+for binpkg_gpg_config in (
+"BINPKG_GPG_SIGNING_GPG_HOME",
+"BINPKG_GPG_SIGNING_KEY",
+):
+if not trees[eroot]["root_config"].settings.get(
+binpkg_gpg_config
+):
+writemsg_level(
+colorize(
+"BAD", f"!!! {binpkg_gpg_config} is not
set\n"
+),
+level=logging.ERROR,
+noiselevel=-1,
+)
+return 1
+
portage.writemsg_stdout(">>> Unlocking GPG... ")
sys.stdout.flush()
gpg = GPG(trees[eroot]["root_config"].settings)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 50854d7a8d7b83e0a598f4e1258d3e063a3fe9e0 Author: Sheng Yu protonmail com> AuthorDate: Fri Sep 16 12:19:19 2022 + Commit: Sam James gentoo org> CommitDate: Tue Sep 20 03:39:22 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=50854d7a Fix wrong creation of empty binpkg from binhost Bug: https://bugs.gentoo.org/870283 Signed-off-by: Sheng Yu protonmail.com> Closes: https://github.com/gentoo/portage/pull/901 Signed-off-by: Sam James gentoo.org> lib/_emerge/Binpkg.py | 9 - 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py index 949ac8ee7..7ce0da15f 100644 --- a/lib/_emerge/Binpkg.py +++ b/lib/_emerge/Binpkg.py @@ -50,6 +50,7 @@ class Binpkg(CompositeTask): "_pkg_path", "_tree", "_verify", +"_pkg_allocated_path", ) def _writemsg_level(self, msg, level=0, noiselevel=0): @@ -68,6 +69,7 @@ class Binpkg(CompositeTask): self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend +self._pkg_allocated_path = None # Use realpath like doebuild_environment() does, since we assert # that this path is literally identical to PORTAGE_BUILDDIR. @@ -206,6 +208,7 @@ class Binpkg(CompositeTask): # --getbinpkg is enabled. if fetcher is not None: self._fetched_pkg = fetcher.pkg_path +self._pkg_allocated_path = fetcher.pkg_allocated_path if self._default_exit(fetcher) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return @@ -246,7 +249,11 @@ class Binpkg(CompositeTask): if self._fetched_pkg: pkg_path = self._bintree.getname( -self._bintree.inject(pkg.cpv, current_pkg_path=self._fetched_pkg), +self._bintree.inject( +pkg.cpv, +current_pkg_path=self._fetched_pkg, +allocated_pkg_path=self._pkg_allocated_path, +), allocate_new=False, ) else:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 47ac5aff464d414bcaa2d04e9eeff06aceec868c Author: Mike Gilbert gentoo org> AuthorDate: Sat Sep 17 23:37:46 2022 + Commit: Mike Gilbert gentoo org> CommitDate: Sun Sep 18 18:33:19 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=47ac5aff Move pty setup from AbstractEbuildProcess to SpawnProcess This ensures that bash is always executed with a controlling terminal, which prevents it from triggering sandbox failures on startup. When merging a package, the 'instprep' phase is executed by a MergeProcess task. The MergeProcess class derives from SpawnProcess directly (not through AbstractEbuildProcess). The SpawnProcess class calls portage.process.spawn() to execute a task. When pid-sandbox is enabled, this creates a new PID namespaces, and starts pid-ns-init to act as PID 1 in the new namespace. pid-ns-init calls setsid(), which creates a new session and disconnects the process from its controlling terminal. Later, it calls ioctl(sys.stdout, termios.TIOCSCTTY) to associate the process with the terminal attached to stdout. This only works if stdout is a tty (not a pipe). If pid-ns-init fails to associate the process with a controlling terminal, bash will fail to open /dev/tty on startup. As a fallback, bash will attempt to dereference /proc/self/fd/0 (stdin), and opens the resulting path with O_RDWR. If the ebuild sets PROPERTIES="interactive", stdin will be inherited from the parent emerge process, and may be attached to a terminal device (/dev/tty1, etc). Attempting to open this device is likely to trigger a sandbox failure. Bug: https://bugs.gentoo.org/870310 Signed-off-by: Mike Gilbert gentoo.org> lib/_emerge/AbstractEbuildProcess.py | 8 lib/_emerge/SpawnProcess.py | 7 ++- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/_emerge/AbstractEbuildProcess.py b/lib/_emerge/AbstractEbuildProcess.py index 6d89d40f0..8712b8ea1 100644 --- a/lib/_emerge/AbstractEbuildProcess.py +++ b/lib/_emerge/AbstractEbuildProcess.py @@ -18,7 +18,6 @@ from portage.package.ebuild._ipc.ExitCommand import ExitCommand from portage.package.ebuild._ipc.QueryCommand import QueryCommand from portage import os from portage.util.futures import asyncio -from portage.util._pty import _create_pty_or_pipe from portage.util import apply_secpass_permissions portage.proxy.lazyimport.lazyimport( @@ -336,13 +335,6 @@ class AbstractEbuildProcess(SpawnProcess): self._eerror(textwrap.wrap(msg, 72)) -def _pipe(self, fd_pipes): -stdout_pipe = None -if not self.background: -stdout_pipe = fd_pipes.get(1) -got_pty, master_fd, slave_fd = _create_pty_or_pipe(copy_term_size=stdout_pipe) -return (master_fd, slave_fd) - def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py index c43b17c12..ed5724c91 100644 --- a/lib/_emerge/SpawnProcess.py +++ b/lib/_emerge/SpawnProcess.py @@ -16,6 +16,7 @@ from portage.output import EOutput from portage.util import writemsg_level from portage.util._async.BuildLogger import BuildLogger from portage.util._async.PipeLogger import PipeLogger +from portage.util._pty import _create_pty_or_pipe from portage.util.futures import asyncio @@ -217,7 +218,11 @@ class SpawnProcess(SubProcess): @type fd_pipes: dict @param fd_pipes: pipes from which to copy terminal size if desired. """ -return os.pipe() +stdout_pipe = None +if not self.background: +stdout_pipe = fd_pipes.get(1) +got_pty, master_fd, slave_fd = _create_pty_or_pipe(copy_term_size=stdout_pipe) +return (master_fd, slave_fd) def _spawn(self, args, **kwargs): spawn_func = portage.process.spawn
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: bb820e4a38cefdb552d02b1d6bcb97eccab38982
Author: Thomas Bracht Laumann Jespersen laumann xyz>
AuthorDate: Mon Aug 8 11:24:48 2022 +
Commit: Sam James gentoo org>
CommitDate: Sat Aug 13 17:56:10 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bb820e4a
EbuildPhase: add SLOT to package version line
Requested as a useful part in the output from tinderbox runs.
Bug: https://bugs.gentoo.org/864382
Signed-off-by: Thomas Bracht Laumann Jespersen laumann.xyz>
Closes: https://github.com/gentoo/portage/pull/885
Signed-off-by: Sam James gentoo.org>
lib/_emerge/EbuildPhase.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 9a04f9c1f..a1eaf86c1 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -135,7 +135,9 @@ class EbuildPhase(CompositeTask):
maint_str = ""
msg = []
-msg.append("Package:%s" % self.settings.mycpv)
+msg.append(
+"Package:%s:%s" % (self.settings.mycpv,
self.settings["SLOT"])
+)
if self.settings.get("PORTAGE_REPO_NAME"):
msg.append("Repository: %s" %
self.settings["PORTAGE_REPO_NAME"])
if maint_str:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 9ab9368c3324d615f1234d1ccb03eae939c393cf Author: John Helmert III gentoo org> AuthorDate: Wed Jun 15 22:11:17 2022 + Commit: Mike Gilbert gentoo org> CommitDate: Fri Jun 17 17:01:49 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9ab9368c depclean: return failure when requested packages aren't depcleaned Closes: https://bugs.gentoo.org/851750 Closes: https://github.com/gentoo/portage/pull/840 Signed-off-by: John Helmert III gentoo.org> Signed-off-by: Mike Gilbert gentoo.org> lib/_emerge/actions.py | 5 + 1 file changed, 5 insertions(+) diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index 0bf2d88b1..e2f3f2ccf 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -1697,6 +1697,11 @@ def _calc_depclean(settings, trees, ldpath_mtimes, myopts, action, args_set, spi cleanlist.append(node.cpv) return _depclean_result(0, cleanlist, ordered, required_pkgs_total, resolver) +if args_set and "--pretend" not in myopts: +# If the cleanlist is empty but we were given packages to clean, +# we aren't successfully depcleaning. Return failure unless +# we're pretending. +return _depclean_result(1, [], False, required_pkgs_total, resolver) return _depclean_result(0, [], False, required_pkgs_total, resolver)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 7fa4fe1b2fddff06b15156c915e7ed6bb29a469e Author: David Palao gmail com> AuthorDate: Tue May 31 15:44:54 2022 + Commit: Mike Gilbert gentoo org> CommitDate: Tue Jun 7 23:47:55 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7fa4fe1b improvement(actions): using new MtimeDB's API to make it read only Signed-off-by: David Palao gmail.com> Signed-off-by: Mike Gilbert gentoo.org> lib/_emerge/actions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index e697459c8..0bf2d88b1 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -263,8 +263,7 @@ def action_build( mergelist_shown = False if pretend or fetchonly: -# make the mtimedb readonly -mtimedb.filename = None +mtimedb.make_readonly() if "--digest" in myopts or "digest" in settings.features: if "--digest" in myopts: msg = "The --digest option"
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: ff27e63569297dcd8b459e7ee255d4335e9f9105
Author: Mike Gilbert gentoo org>
AuthorDate: Fri Apr 22 21:13:11 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Apr 22 21:20:32 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ff27e635
MiscFunctionsProcess: use a separate SANDBOX_LOG
This avoids clobbering the main sandbox.log file when
"misc-functions.sh die_hooks" is invoked after an ebuild phase.
Signed-off-by: Mike Gilbert gentoo.org>
lib/_emerge/MiscFunctionsProcess.py | 11 +++
1 file changed, 11 insertions(+)
diff --git a/lib/_emerge/MiscFunctionsProcess.py
b/lib/_emerge/MiscFunctionsProcess.py
index 7a7b1f5ac..16fd08222 100644
--- a/lib/_emerge/MiscFunctionsProcess.py
+++ b/lib/_emerge/MiscFunctionsProcess.py
@@ -48,9 +48,20 @@ class MiscFunctionsProcess(AbstractEbuildProcess):
# Temporarily unset EBUILD_PHASE so that bashrc code doesn't
# think this is a real phase.
phase_backup = self.settings.pop("EBUILD_PHASE", None)
+
+# Use a separate sandbox log file to avoid clobbering the real one.
+sandbox_log = self.settings.pop("SANDBOX_LOG", None)
+if sandbox_log is not None:
+self.settings["SANDBOX_LOG"] = os.path.join(
+self.settings["T"], "sandbox-misc.log"
+)
+
try:
return spawn(" ".join(args), self.settings, **kwargs)
finally:
if phase_backup is not None:
self.settings["EBUILD_PHASE"] = phase_backup
self.settings.pop("PORTAGE_PIPE_FD", None)
+self.settings.pop("SANDBOX_LOG", None)
+if sandbox_log is not None:
+self.settings["SANDBOX_LOG"] = sandbox_log
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 195b8029ddd510188e8a2510dad5cbce67c44ecc
Author: Kenneth Raplee kennethraplee com>
AuthorDate: Fri Mar 25 12:41:54 2022 +
Commit: Sam James gentoo org>
CommitDate: Sun Mar 27 23:06:46 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=195b8029
Consolidate checks for options with bad atoms
Signed-off-by: Kenneth Raplee kennethraplee.com>
Signed-off-by: Sam James gentoo.org>
lib/_emerge/main.py | 57 +
1 file changed, 14 insertions(+), 43 deletions(-)
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index f3ff02404..01dc1b419 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -876,52 +876,23 @@ def parse_opts(tmpcmdline, silent=False):
if myoptions.depclean_lib_check in true_y:
myoptions.depclean_lib_check = True
-if myoptions.exclude:
-bad_atoms = _find_bad_atoms(myoptions.exclude)
-if bad_atoms and not silent:
-parser.error(
-"Invalid Atom(s) in --exclude parameter: '%s' (only package
names and slot atoms (with wildcards) allowed)\n"
-% (",".join(bad_atoms),)
-)
-
-if myoptions.reinstall_atoms:
-bad_atoms = _find_bad_atoms(myoptions.reinstall_atoms)
-if bad_atoms and not silent:
-parser.error(
-"Invalid Atom(s) in --reinstall-atoms parameter: '%s' (only
package names and slot atoms (with wildcards) allowed)\n"
-% (",".join(bad_atoms),)
-)
-
-if myoptions.rebuild_exclude:
-bad_atoms = _find_bad_atoms(myoptions.rebuild_exclude)
-if bad_atoms and not silent:
-parser.error(
-"Invalid Atom(s) in --rebuild-exclude parameter: '%s' (only
package names and slot atoms (with wildcards) allowed)\n"
-% (",".join(bad_atoms),)
-)
-
-if myoptions.rebuild_ignore:
-bad_atoms = _find_bad_atoms(myoptions.rebuild_ignore)
-if bad_atoms and not silent:
-parser.error(
-"Invalid Atom(s) in --rebuild-ignore parameter: '%s' (only
package names and slot atoms (with wildcards) allowed)\n"
-% (",".join(bad_atoms),)
-)
-
-if myoptions.usepkg_exclude:
-bad_atoms = _find_bad_atoms(myoptions.usepkg_exclude)
-if bad_atoms and not silent:
-parser.error(
-"Invalid Atom(s) in --usepkg-exclude parameter: '%s' (only
package names and slot atoms (with wildcards) allowed)\n"
-% (",".join(bad_atoms),)
-)
+candidate_bad_options = (
+(myoptions.exclude, "exclude"),
+(myoptions.reinstall_atoms, "reinstall-atoms"),
+(myoptions.rebuild_exclude, "rebuild-exclude"),
+(myoptions.rebuild_ignore, "rebuild-ignore"),
+(myoptions.usepkg_exclude, "usepkg-exclude"),
+(myoptions.useoldpkg_atoms, "useoldpkg-atoms"),
+)
+bad_options = (
+(_find_bad_atoms(atoms), flag) for atoms, flag in
candidate_bad_options if atoms
+)
-if myoptions.useoldpkg_atoms:
-bad_atoms = _find_bad_atoms(myoptions.useoldpkg_atoms)
+for bad_atoms, flag in bad_options:
if bad_atoms and not silent:
+invalid_atoms = ",".join(bad_atoms)
parser.error(
-"Invalid Atom(s) in --useoldpkg-atoms parameter: '%s' (only
package names and slot atoms (with wildcards) allowed)\n"
-% (",".join(bad_atoms),)
+f"Invalid Atom(s) in --{flag} parameter: '{invalid_atoms}'
(only package names and slot atoms (with wildcards) allowed)\n"
)
if myoptions.fail_clean in true_y:
