[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/git/
commit: 95b4337d376a146db8fda7717393366175cbd285
Author: Alex Xu (Hello71) yahoo ca>
AuthorDate: Sun May 5 20:24:52 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun May 12 16:20:02 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=95b4337d
sync: git: add safe.directory for get head commit
Closes: https://bugs.gentoo.org/930992
Fixes: 1339a02103 ("sync: git: include signing key and git revision in log
output")
Signed-off-by: Alex Xu (Hello71) yahoo.ca>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/git/git.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/portage/sync/modules/git/git.py
b/lib/portage/sync/modules/git/git.py
index 8fdbf97de0..a2830280fb 100644
--- a/lib/portage/sync/modules/git/git.py
+++ b/lib/portage/sync/modules/git/git.py
@@ -606,6 +606,7 @@ class GitSync(NewBase):
if self.bin_command is None:
# return quietly so that we don't pollute emerge --info output
return (1, False)
+self.add_safe_directory()
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
try:
ret = (
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: 8c6e5d06afbf6fca1893cff5ed777e44f93b7a5d
Author: Alexey Gladkov kernel org>
AuthorDate: Sun Mar 3 16:41:08 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8c6e5d06
sync/zipfile: Handle ETag header
Most services add an ETag header and determine whether the locally
cached version of the URL has expired. So we can add ETag processing to
avoid unnecessary downloading and unpacking of the zip archive.
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/zipfile.py | 36 +++--
1 file changed, 29 insertions(+), 7 deletions(-)
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
index 1762d2c8f1..bb78b39243 100644
--- a/lib/portage/sync/modules/zipfile/zipfile.py
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -10,7 +10,7 @@ import tempfile
import urllib.request
import portage
-from portage.util import writemsg_level
+from portage.util import writemsg_level, writemsg_stdout
from portage.sync.syncbase import SyncBase
@@ -31,13 +31,31 @@ class ZipFile(SyncBase):
if kwargs:
self._kwargs(kwargs)
-# initial checkout
-zip_uri = self.repo.sync_uri
+req = urllib.request.Request(url=self.repo.sync_uri)
-with urllib.request.urlopen(zip_uri) as response:
-with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
-shutil.copyfileobj(response, tmp_file)
-zip_file = tmp_file.name
+info = portage.grabdict(os.path.join(self.repo.location, ".info"))
+if "etag" in info:
+req.add_header("If-None-Match", info["etag"][0])
+
+try:
+with urllib.request.urlopen(req) as response:
+with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+shutil.copyfileobj(response, tmp_file)
+
+zip_file = tmp_file.name
+etag = response.headers.get("etag")
+
+except urllib.error.HTTPError as resp:
+if resp.code == 304:
+writemsg_stdout(">>> The repository has not changed.\n",
noiselevel=-1)
+return (os.EX_OK, False)
+
+writemsg_level(
+f"!!! Unable to obtain zip archive: {resp}\n",
+noiselevel=-1,
+level=logging.ERROR,
+)
+return (1, False)
if not zipfile.is_zipfile(zip_file):
msg = "!!! file is not a zip archive."
@@ -77,6 +95,10 @@ class ZipFile(SyncBase):
with open(dstpath, "wb") as dstfile:
shutil.copyfileobj(srcfile, dstfile)
+with open(os.path.join(self.repo.location, ".info"), "w") as infofile:
+if etag:
+infofile.write(f"etag {etag}\n")
+
os.unlink(zip_file)
return (os.EX_OK, True)
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: 7e93192fda22594b9e9d223c54a39b4bad0554f9
Author: Alexey Gladkov kernel org>
AuthorDate: Mon Mar 11 00:25:07 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7e93192f
sync/zipfile: Add retrieve_head to return archive checksum
Since we have an ETag, we can return the checksum of the archive. It
will be a replacement for the head commit of the repository.
Suggested-by: Zac Medico gentoo.org>
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/__init__.py | 3 ++-
lib/portage/sync/modules/zipfile/zipfile.py | 9 +
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/lib/portage/sync/modules/zipfile/__init__.py
b/lib/portage/sync/modules/zipfile/__init__.py
index 19fe3af412..e44833088c 100644
--- a/lib/portage/sync/modules/zipfile/__init__.py
+++ b/lib/portage/sync/modules/zipfile/__init__.py
@@ -21,10 +21,11 @@ module_spec = {
"sourcefile": "zipfile",
"class": "ZipFile",
"description": doc,
-"functions": ["sync"],
+"functions": ["sync", "retrieve_head"],
"func_desc": {
"sync": "Performs an archived http download of the "
+ "repository, then unpacks it.",
+"retrieve_head": "Returns the checksum of the unpacked
archive.",
},
"validate_config": CheckSyncConfig,
"module_specific_options": (),
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
index bb78b39243..3cd210a64b 100644
--- a/lib/portage/sync/modules/zipfile/zipfile.py
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -26,6 +26,15 @@ class ZipFile(SyncBase):
def __init__(self):
SyncBase.__init__(self, "emerge", ">=sys-apps/portage-2.3")
+def retrieve_head(self, **kwargs):
+"""Get information about the checksum of the unpacked archive"""
+if kwargs:
+self._kwargs(kwargs)
+info = portage.grabdict(os.path.join(self.repo.location, ".info"))
+if "etag" in info:
+return (os.EX_OK, info["etag"][0])
+return (1, False)
+
def sync(self, **kwargs):
"""Sync the repository"""
if kwargs:
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: ced2e6d4f4ac95b8e17cf7dae964a64037a85bf0
Author: Alexey Gladkov kernel org>
AuthorDate: Mon Mar 11 17:09:05 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:09 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ced2e6d4
sync/zipfile: Recycle files that have not changed
We can check whether the content of files from the archive differs from
the current revision. This will give us several advantages:
* This will give us some meaning to the mtime of files, since it will
prevent the timestamps of unmodified files from being changed.
* This will also get rid of recreatiing self.repo.location, which will
allow sync with FEATURES=usersync because self.repo.location is reused.
Suggested-by: Zac Medico gentoo.org>
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/zipfile.py | 32 -
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
index 3cd210a64b..edfb5aa681 100644
--- a/lib/portage/sync/modules/zipfile/zipfile.py
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -35,6 +35,16 @@ class ZipFile(SyncBase):
return (os.EX_OK, info["etag"][0])
return (1, False)
+def _do_cmp(self, f1, f2):
+bufsize = 8 * 1024
+while True:
+b1 = f1.read(bufsize)
+b2 = f2.read(bufsize)
+if b1 != b2:
+return False
+if not b1:
+return True
+
def sync(self, **kwargs):
"""Sync the repository"""
if kwargs:
@@ -76,7 +86,15 @@ class ZipFile(SyncBase):
return (1, False)
# Drop previous tree
-shutil.rmtree(self.repo.location)
+tempdir = tempfile.mkdtemp(prefix=".temp", dir=self.repo.location)
+tmpname = os.path.basename(tempdir)
+
+for name in os.listdir(self.repo.location):
+if name != tmpname:
+os.rename(
+os.path.join(self.repo.location, name),
+os.path.join(tempdir, name),
+)
with zipfile.ZipFile(zip_file) as archive:
strip_comp = 0
@@ -101,9 +119,21 @@ class ZipFile(SyncBase):
continue
with archive.open(n) as srcfile:
+prvpath = os.path.join(tempdir, *parts[strip_comp:])
+
+if os.path.exists(prvpath):
+with open(prvpath, "rb") as prvfile:
+if self._do_cmp(prvfile, srcfile):
+os.rename(prvpath, dstpath)
+continue
+srcfile.seek(0)
+
with open(dstpath, "wb") as dstfile:
shutil.copyfileobj(srcfile, dstfile)
+# Drop previous tree
+shutil.rmtree(tempdir)
+
with open(os.path.join(self.repo.location, ".info"), "w") as infofile:
if etag:
infofile.write(f"etag {etag}\n")
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/sync/
commit: b01cd4208a17a141311d490788aff11537312575
Author: Zac Medico gentoo org>
AuthorDate: Tue Mar 12 10:20:56 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:09 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b01cd420
sync/zipfile: Add testcase for etag
Signed-off-by: Zac Medico gentoo.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/tests/sync/meson.build | 1 +
lib/portage/tests/sync/test_sync_zipfile.py | 99 +
2 files changed, 100 insertions(+)
diff --git a/lib/portage/tests/sync/meson.build
b/lib/portage/tests/sync/meson.build
index b78583021f..8c566080e3 100644
--- a/lib/portage/tests/sync/meson.build
+++ b/lib/portage/tests/sync/meson.build
@@ -1,6 +1,7 @@
py.install_sources(
[
'test_sync_local.py',
+'test_sync_zipfile.py',
'__init__.py',
'__test__.py',
],
diff --git a/lib/portage/tests/sync/test_sync_zipfile.py
b/lib/portage/tests/sync/test_sync_zipfile.py
new file mode 100644
index 00..4fbde8a351
--- /dev/null
+++ b/lib/portage/tests/sync/test_sync_zipfile.py
@@ -0,0 +1,99 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import http.server
+import os
+import shutil
+import socketserver
+import subprocess
+import tempfile
+import textwrap
+import threading
+from functools import partial
+
+import portage
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import ResolverPlayground
+
+
+class test_sync_zipfile_case(TestCase):
+def test_sync_zipfile(self):
+cpv = "dev-libs/A-0"
+ebuilds = {
+cpv: {"EAPI": "8"},
+}
+etag = "foo"
+
+server = None
+playground = None
+tmpdir = tempfile.mkdtemp()
+try:
+
+class Handler(http.server.SimpleHTTPRequestHandler):
+def end_headers(self):
+self.send_header("etag", etag)
+super().end_headers()
+
+server = socketserver.TCPServer(
+("127.0.0.1", 0),
+partial(Handler, directory=tmpdir),
+)
+threading.Thread(target=server.serve_forever, daemon=True).start()
+
+playground = ResolverPlayground(
+ebuilds=ebuilds,
+)
+settings = playground.settings
+
+env = settings.environ()
+
+repos_conf = textwrap.dedent(
+"""
+[test_repo]
+location = %(location)s
+sync-type = zipfile
+sync-uri = %(sync-uri)s
+auto-sync = true
+"""
+)
+
+repo_location = f"{playground.eprefix}/var/repositories/test_repo"
+
+env["PORTAGE_REPOSITORIES"] = repos_conf % {
+"location": repo_location,
+"sync-uri":
"http://{}:{}/test_repo.zip".format(*server.server_address),
+}
+
+shutil.make_archive(os.path.join(tmpdir, "test_repo"), "zip",
repo_location)
+
+ebuild =
playground.trees[playground.eroot]["porttree"].dbapi.findname(cpv)
+self.assertTrue(os.path.exists(ebuild))
+shutil.rmtree(repo_location)
+self.assertFalse(os.path.exists(ebuild))
+
+result = subprocess.run(
+[
+"emerge",
+"--sync",
+],
+env=env,
+stdout=subprocess.PIPE,
+stderr=subprocess.STDOUT,
+)
+output = result.stdout.decode(errors="replace")
+try:
+self.assertEqual(result.returncode, os.EX_OK)
+except Exception:
+print(output)
+raise
+
+repo = settings.repositories["test_repo"]
+sync_mod = portage.sync.module_controller.get_class("zipfile")
+status, repo_revision = sync_mod().retrieve_head(options={"repo":
repo})
+self.assertEqual(status, os.EX_OK)
+self.assertEqual(repo_revision, etag)
+finally:
+if server is not None:
+server.shutdown()
+shutil.rmtree(tmpdir)
+playground.cleanup()
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: 80445d9b00bfcd1eb4955cf3ecb397b4c02663ba
Author: Alexey Gladkov kernel org>
AuthorDate: Mon Feb 12 13:59:40 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=80445d9b
sync: Add method to download zip archives
Add a simple method for synchronizing repository as a snapshot in a zip
archive. The implementation does not require external utilities to
download and unpack archive. This makes the method very cheap.
The main usecase being considered is obtaining snapshots of github
repositories, but many other web interfaces for git also support
receiving snapshots in zip format.
For example, to get a snapshot of the master branch:
https://github.com/gentoo/portage/archive/refs/heads/master.zip
https://gitweb.gentoo.org/proj/portage.git/snapshot/portage-master.zip
or a link to a snapshot of the tag:
https://github.com/gentoo/portage/archive/refs/tags/portage-3.0.61.zip
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/__init__.py | 33 +++
lib/portage/sync/modules/zipfile/zipfile.py | 82
2 files changed, 115 insertions(+)
diff --git a/lib/portage/sync/modules/zipfile/__init__.py
b/lib/portage/sync/modules/zipfile/__init__.py
new file mode 100644
index 00..19fe3af412
--- /dev/null
+++ b/lib/portage/sync/modules/zipfile/__init__.py
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2024 Alexey Gladkov
+
+doc = """Zipfile plug-in module for portage.
+Performs a http download of a portage snapshot and unpacks it to the repo
+location."""
+__doc__ = doc[:]
+
+
+import os
+
+from portage.sync.config_checks import CheckSyncConfig
+
+
+module_spec = {
+"name": "zipfile",
+"description": doc,
+"provides": {
+"zipfile-module": {
+"name": "zipfile",
+"sourcefile": "zipfile",
+"class": "ZipFile",
+"description": doc,
+"functions": ["sync"],
+"func_desc": {
+"sync": "Performs an archived http download of the "
++ "repository, then unpacks it.",
+},
+"validate_config": CheckSyncConfig,
+"module_specific_options": (),
+},
+},
+}
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
new file mode 100644
index 00..1762d2c8f1
--- /dev/null
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2024 Alexey Gladkov
+
+import os
+import os.path
+import logging
+import zipfile
+import shutil
+import tempfile
+import urllib.request
+
+import portage
+from portage.util import writemsg_level
+from portage.sync.syncbase import SyncBase
+
+
+class ZipFile(SyncBase):
+"""ZipFile sync module"""
+
+short_desc = "Perform sync operations on GitHub repositories"
+
+@staticmethod
+def name():
+return "ZipFile"
+
+def __init__(self):
+SyncBase.__init__(self, "emerge", ">=sys-apps/portage-2.3")
+
+def sync(self, **kwargs):
+"""Sync the repository"""
+if kwargs:
+self._kwargs(kwargs)
+
+# initial checkout
+zip_uri = self.repo.sync_uri
+
+with urllib.request.urlopen(zip_uri) as response:
+with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+shutil.copyfileobj(response, tmp_file)
+zip_file = tmp_file.name
+
+if not zipfile.is_zipfile(zip_file):
+msg = "!!! file is not a zip archive."
+self.logger(self.xterm_titles, msg)
+writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
+
+os.unlink(zip_file)
+
+return (1, False)
+
+# Drop previous tree
+shutil.rmtree(self.repo.location)
+
+with zipfile.ZipFile(zip_file) as archive:
+strip_comp = 0
+
+for f in archive.namelist():
+f = os.path.normpath(f)
+if os.path.basename(f) == "profiles":
+strip_comp = f.count("/")
+break
+
+for n in archive.infolist():
+p = os.path.normpath(n.filename)
+
+if os.path.isabs(p):
+continue
+
+parts = p.split("/")
+dstpath = os.path.join(self.repo.location, *parts[strip_comp:])
+
+if n.is_dir():
+os.makedirs(dstpath, mode=0o755, exist_ok=True)
+continue
+
+with archive.open(n) as srcfile:
+with open(dstpath, "wb") as dstfile:
+shutil.copyfileobj(srcfile, dstfile)
+
+os.unlink(zip_file)
+
+return (os.EX_OK, True)
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 36235596e061bf8cf4729b3915f9aaa6ec80baa3
Author: Alfred Wingate protonmail com>
AuthorDate: Sat Apr 6 08:28:43 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Apr 26 22:05:48 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=36235596
lib: adhere to python package version conventions
* Commit metadata isn't valid version that python tooling is expected to
parse. Follow python ecosystem conventions and make it a local
version.
https://packaging.python.org/en/latest/specifications/version-specifiers/#local-version-segments
Example:
* Old: 3.0.63-g08a2bc380
* New: 3.0.63+g08a2bc380
Bug: https://bugs.gentoo.org/926966
Signed-off-by: Alfred Wingate protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1314
Signed-off-by: Sam James gentoo.org>
lib/portage/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index a468eeaff3..21bf993170 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -732,7 +732,7 @@ if installation.TYPE == installation.TYPES.SOURCE:
output = _unicode_decode(proc.communicate()[0],
encoding=encoding)
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
-VERSION = output.lstrip('portage-').strip()
+VERSION = output.lstrip("portage-").strip().replace("-g",
"+g")
else:
VERSION = "HEAD"
return VERSION
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 381fad5e3554ec94ec5626e8c17874f32b30b752
Author: Sam James gentoo org>
AuthorDate: Tue Aug 29 07:26:36 2023 +
Commit: Sam James gentoo org>
CommitDate: Fri Apr 26 22:05:48 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=381fad5e
lib: use more pure git-describe output for --version
Use `git describe --dirty` output rather than mangling git-describe and
reinventing
--dirty by manually checking for changes post-commit.
We no longer mangle the 7th commit post-tag into _p7, but instead do:
${tag}-7-${last_commit}.
This is similar to gnulib's git-version-gen (which we may still want to import,
not sure, this seems enough for now) and is familiar output for developers.
Example:
* Old: 3.0.51_p7
* New: 3.0.51-7-g098b30548
Bug: https://bugs.gentoo.org/912209
Signed-off-by: Sam James gentoo.org>
lib/portage/__init__.py | 35 ---
1 file changed, 4 insertions(+), 31 deletions(-)
diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index aa81bdb4c2..a468eeaff3 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -720,10 +720,7 @@ if installation.TYPE == installation.TYPES.SOURCE:
BASH_BINARY,
"-c",
(
-f"cd {_shell_quote(PORTAGE_BASE_PATH)} ; git describe
--match 'portage-*' || exit $? ; "
-'if [ -n "`git diff-index --name-only --diff-filter=M
HEAD`" ] ; '
-"then echo modified ; git rev-list --format=%%ct -n 1
HEAD ; fi ; "
-"exit 0"
+f"cd {_shell_quote(PORTAGE_BASE_PATH)} ; git describe
--dirty --match 'portage-*' || exit $? ; "
),
]
cmd = [
@@ -735,33 +732,9 @@ if installation.TYPE == installation.TYPES.SOURCE:
output = _unicode_decode(proc.communicate()[0],
encoding=encoding)
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
-output_lines = output.splitlines()
-if output_lines:
-version_split = output_lines[0].split("-")
-if len(version_split) > 1:
-VERSION = version_split[1]
-patchlevel = False
-if len(version_split) > 2:
-patchlevel = True
-VERSION = f"{VERSION}_p{version_split[2]}"
-if len(output_lines) > 1 and output_lines[1] ==
"modified":
-head_timestamp = None
-if len(output_lines) > 3:
-try:
-head_timestamp = int(output_lines[3])
-except ValueError:
-pass
-timestamp = int(time.time())
-if (
-head_timestamp is not None
-and timestamp > head_timestamp
-):
-timestamp = timestamp - head_timestamp
-if not patchlevel:
-VERSION = f"{VERSION}_p0"
-VERSION = f"{VERSION}_p{timestamp}"
-return VERSION
-VERSION = "HEAD"
+VERSION = output.lstrip('portage-').strip()
+else:
+VERSION = "HEAD"
return VERSION
VERSION = _LazyVersion()
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/
commit: 72f41d07b5396195a98691fafeb3e5b207a76564 Author: Raul E Rangel chromium org> AuthorDate: Thu Mar 21 20:29:59 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 24 22:18:34 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=72f41d07 Reapply "config: Don't directly modify FEATURES" This reverts commit b150419d28bd7afb98404a829c639584d34efc03. It turns out we need to keep the open coded version to avoid creating a persistent setting. See https://github.com/gentoo/portage/pull/1098#issuecomment-1761638611 This change just adds a sorted() so we get deterministic ordering. Bug: https://bugs.gentoo.org/914441 Signed-off-by: Raul E Rangel chromium.org> Closes: https://github.com/gentoo/portage/pull/1312 Signed-off-by: Zac Medico gentoo.org> lib/portage/package/ebuild/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py index bafdc55a08..67fd1bb18d 100644 --- a/lib/portage/package/ebuild/config.py +++ b/lib/portage/package/ebuild/config.py @@ -2206,7 +2206,9 @@ class config: # "test" is in IUSE and USE=test is masked, so execution # of src_test() probably is not reliable. Therefore, # temporarily disable FEATURES=test just for this package. -self["FEATURES"] = " ".join(x for x in self.features if x != "test") +self["FEATURES"] = " ".join( +x for x in sorted(self.features) if x != "test" +) # Allow _* flags from USE_EXPAND wildcards to pass through here. use.difference_update(
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/util/file_copy/, src/
commit: 23529ee81964665107400e87fc3d49c256e915c0
Author: Mike Gilbert gentoo org>
AuthorDate: Fri Mar 1 15:45:58 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 15 20:05:34 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=23529ee8
Replace linux_reflink extension module
Python 3.8 added support for copy_file_range in the os module,
so we can just call that directly.
Also, we can use the FICLONE ioctl for fast file clones on supported
filesystems (btrfs).
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/file_copy.py | 137 ++
lib/portage/util/file_copy/__init__.py | 36 ---
lib/portage/util/file_copy/meson.build | 7 -
lib/portage/util/meson.build | 2 +-
src/meson.build| 20 --
src/portage_util_file_copy_reflink_linux.c | 396 -
6 files changed, 138 insertions(+), 460 deletions(-)
diff --git a/lib/portage/util/file_copy.py b/lib/portage/util/file_copy.py
new file mode 100644
index 00..e3926d8ef6
--- /dev/null
+++ b/lib/portage/util/file_copy.py
@@ -0,0 +1,137 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import fcntl
+import logging
+import os
+import platform
+import shutil
+import sys
+
+
+logger = logging.getLogger(__name__)
+
+# Added in Python 3.12
+FICLONE = getattr(fcntl, "FICLONE", 0x40049409)
+
+# Unavailable in PyPy
+SEEK_DATA = getattr(os, "SEEK_DATA", 3)
+SEEK_HOLE = getattr(os, "SEEK_HOLE", 4)
+
+
+def _get_chunks(src):
+try:
+offset_hole = 0
+while True:
+try:
+# Find the next bit of data
+offset_data = os.lseek(src, offset_hole, SEEK_DATA)
+except OSError as e:
+# Re-raise for unexpected errno values
+if e.errno not in (errno.EINVAL, errno.ENXIO):
+raise
+
+offset_end = os.lseek(src, 0, os.SEEK_END)
+
+if e.errno == errno.ENXIO:
+# End of file
+if offset_end > offset_hole:
+# Hole at end of file
+yield (offset_end, 0)
+else:
+# SEEK_DATA failed with EINVAL, return the whole file
+yield (0, offset_end)
+
+break
+else:
+offset_hole = os.lseek(src, offset_data, SEEK_HOLE)
+yield (offset_data, offset_hole - offset_data)
+
+except OSError:
+logger.warning("_get_chunks failed unexpectedly",
exc_info=sys.exc_info())
+raise
+
+
+def _do_copy_file_range(src, dst, offset, count):
+while count > 0:
+# count must fit in ssize_t
+c = min(count, sys.maxsize)
+written = os.copy_file_range(src, dst, c, offset, offset)
+if written == 0:
+# https://bugs.gentoo.org/828844
+raise OSError(errno.EOPNOTSUPP, os.strerror(errno.EOPNOTSUPP))
+offset += written
+count -= written
+
+
+def _do_sendfile(src, dst, offset, count):
+os.lseek(dst, offset, os.SEEK_SET)
+while count > 0:
+# count must fit in ssize_t
+c = min(count, sys.maxsize)
+written = os.sendfile(dst, src, offset, c)
+offset += written
+count -= written
+
+
+def _fastcopy(src, dst):
+with (
+open(src, "rb", buffering=0) as srcf,
+open(dst, "wb", buffering=0) as dstf,
+):
+srcfd = srcf.fileno()
+dstfd = dstf.fileno()
+
+if platform.system() == "Linux":
+try:
+fcntl.ioctl(dstfd, FICLONE, srcfd)
+return
+except OSError:
+pass
+
+try_cfr = hasattr(os, "copy_file_range")
+
+for offset, count in _get_chunks(srcfd):
+if count == 0:
+os.ftruncate(dstfd, offset)
+else:
+if try_cfr:
+try:
+_do_copy_file_range(srcfd, dstfd, offset, count)
+continue
+except OSError as e:
+try_cfr = False
+if e.errno not in (errno.EXDEV, errno.ENOSYS,
errno.EOPNOTSUPP):
+logger.warning(
+"_do_copy_file_range failed unexpectedly",
+exc_info=sys.exc_info(),
+)
+try:
+_do_sendfile(srcfd, dstfd, offset, count)
+except OSError:
+logger.warning(
+"_do_sendfile failed unexpectedly",
exc_info=sys.exc_info()
+)
+raise
+
+
+def copyfile(src, dst):
+"""
+Copy the contents (no metadata) of the file named src to a file
+named dst.
+
+If possible,
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/file_copy/
commit: 0cc3f9e269b26173c2f81d731c5fe208b758270b
Author: Mike Gilbert gentoo org>
AuthorDate: Sun Mar 3 02:28:42 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 15 20:05:34 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0cc3f9e2
Improve testCopyFileSparse
Actually create sparse blocks at the start and end.
Check file size before/after copying.
Ensure sparse output when _fastcopy succeeds.
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/tests/util/file_copy/test_copyfile.py | 35 ---
1 file changed, 24 insertions(+), 11 deletions(-)
diff --git a/lib/portage/tests/util/file_copy/test_copyfile.py
b/lib/portage/tests/util/file_copy/test_copyfile.py
index e91a47bed8..e114e6ae35 100644
--- a/lib/portage/tests/util/file_copy/test_copyfile.py
+++ b/lib/portage/tests/util/file_copy/test_copyfile.py
@@ -3,13 +3,14 @@
import shutil
import tempfile
+from unittest.mock import patch
import pytest
from portage import os
from portage.tests import TestCase
from portage.checksum import perform_md5
-from portage.util.file_copy import copyfile
+from portage.util.file_copy import copyfile, _fastcopy
class CopyFileTestCase(TestCase):
@@ -42,25 +43,37 @@ class CopyFileSparseTestCase(TestCase):
# files too big, in case the filesystem doesn't support
# sparse files.
with open(src_path, "wb") as f:
+f.seek(2**16, os.SEEK_SET)
f.write(content)
-f.seek(2**17, 1)
-f.write(content)
-f.seek(2**18, 1)
+f.seek(2**17, os.SEEK_SET)
f.write(content)
# Test that sparse blocks are handled correctly at
-# the end of the file (involves seek and truncate).
-f.seek(2**17, 1)
+# the end of the file.
+f.truncate(2**18)
-copyfile(src_path, dest_path)
+fastcopy_success = False
+
+def mock_fastcopy(src, dst):
+nonlocal fastcopy_success
+_fastcopy(src, dst)
+fastcopy_success = True
+
+with patch("portage.util.file_copy._fastcopy", new=mock_fastcopy):
+copyfile(src_path, dest_path)
self.assertEqual(perform_md5(src_path), perform_md5(dest_path))
-# This last part of the test is expected to fail when sparse
-# copy is not implemented, so mark it xfail:
-pytest.xfail(reason="sparse copy is not implemented")
+src_stat = os.stat(src_path)
+dest_stat = os.stat(dest_path)
+
+self.assertEqual(src_stat.st_size, dest_stat.st_size)
# If sparse blocks were preserved, then both files should
# consume the same number of blocks.
-self.assertEqual(os.stat(src_path).st_blocks,
os.stat(dest_path).st_blocks)
+# This is expected to fail when sparse copy is not implemented.
+if src_stat.st_blocks != dest_stat.st_blocks:
+if fastcopy_success:
+pytest.fail(reason="sparse copy failed with _fastcopy")
+pytest.xfail(reason="sparse copy is not implemented")
finally:
shutil.rmtree(tempdir)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: b7e89f866a9a1d73ab72670d74e2292b05893849
Author: Mike Gilbert gentoo org>
AuthorDate: Wed Mar 6 04:01:27 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Wed Mar 6 18:19:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7e89f86
util: set a timeout for urlopen calls
A hung urlopen call can cause emerge to produce no output when fetching
binhost data.
Bug: https://bugs.gentoo.org/926221
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/_urlopen.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/portage/util/_urlopen.py b/lib/portage/util/_urlopen.py
index 22f0e08df0..d451a94a89 100644
--- a/lib/portage/util/_urlopen.py
+++ b/lib/portage/util/_urlopen.py
@@ -26,10 +26,10 @@ def have_pep_476():
return hasattr(__import__("ssl"), "_create_unverified_context")
-def urlopen(url, if_modified_since=None, headers={}, proxies=None):
+def urlopen(url, timeout=10, if_modified_since=None, headers={}, proxies=None):
parse_result = urllib_parse.urlparse(url)
if parse_result.scheme not in ("http", "https"):
-return _urlopen(url)
+return _urlopen(url, timeout=timeout)
netloc = parse_result.netloc.rpartition("@")[-1]
url = urllib_parse.urlunparse(
@@ -59,7 +59,7 @@ def urlopen(url, if_modified_since=None, headers={},
proxies=None):
handlers.append(urllib_request.ProxyHandler(proxies))
opener = urllib_request.build_opener(*handlers)
-hdl = opener.open(request)
+hdl = opener.open(request, timeout=timeout)
if hdl.headers.get("last-modified", ""):
try:
add_header = hdl.headers.add_header
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/, lib/portage/util/, lib/portage/tests/util/
commit: 11c65496bd951c3b7778a3c1ea240e347b1f4c38 Author: Zac Medico gentoo org> AuthorDate: Sun Mar 3 21:06:13 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 3 21:10:49 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=11c65496 socks5: Use run_coroutine_exitfuncs() Since commit c3ebdbb42e72 the atexit_register(proxy.stop) call in the get_socks5_proxy function can accept a coroutine function to execute in run_coroutine_exitfuncs(), so convert the ProxyManager stop method to a coroutine and use run_coroutine_exitfuncs(). Bug: https://bugs.gentoo.org/925240 Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/test_socks5.py | 5 ++- lib/portage/util/_eventloop/asyncio_event_loop.py | 12 --- lib/portage/util/socks5.py| 39 +++ 3 files changed, 7 insertions(+), 49 deletions(-) diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py index 4a6d08169d..a8cd0c46c4 100644 --- a/lib/portage/tests/util/test_socks5.py +++ b/lib/portage/tests/util/test_socks5.py @@ -216,10 +216,9 @@ class Socks5ServerTestCase(TestCase): self.assertEqual(result, content) finally: try: -# Also run_exitfuncs to test atexit hook cleanup. -await socks5.proxy.stop() +# Also run_coroutine_exitfuncs to test atexit hook cleanup. self.assertNotEqual(portage.process._exithandlers, []) -portage.process.run_exitfuncs() +await portage.process.run_coroutine_exitfuncs() self.assertEqual(portage.process._exithandlers, []) finally: portage.process._exithandlers = previous_exithandlers diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index a598b1b516..821cc7f102 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -15,7 +15,6 @@ except ImportError: PidfdChildWatcher = None import portage -from portage.util import socks5 class AsyncioEventLoop(_AbstractEventLoop): @@ -75,17 +74,6 @@ class AsyncioEventLoop(_AbstractEventLoop): self._closing = False async def _close_main(self): -# Even though this has an exit hook, invoke it here so that -# we can properly wait for it and avoid messages like this: -# [ERROR] Task was destroyed but it is pending! -if socks5.proxy.is_running(): -# TODO: Convert socks5.proxy.stop() to a regular coroutine -# function so that it doesn't need to be wrapped like this. -async def stop_socks5_proxy(): -await socks5.proxy.stop() - -portage.process.atexit_register(stop_socks5_proxy) - await portage.process.run_coroutine_exitfuncs() portage.process.run_exitfuncs() diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index f8fcdf9fca..c32ba77674 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -6,15 +6,8 @@ import asyncio import errno import os import socket -from typing import Union import portage - -portage.proxy.lazyimport.lazyimport( -globals(), -"portage.util._eventloop.global_event_loop:global_event_loop", -) - import portage.data from portage import _python_interpreter from portage.data import portage_gid, portage_uid, userpriv_groups @@ -65,41 +58,19 @@ class ProxyManager: **spawn_kwargs, ) -def stop(self) -> Union[None, asyncio.Future]: +async def stop(self): """ -Stop the SOCKSv5 server. - -If there is a running asyncio event loop then asyncio.Future is -returned which should be used to wait for the server process -to exit. +Stop the SOCKSv5 server. This method is a coroutine. """ -future = None -try: -loop = asyncio.get_running_loop() -except RuntimeError: -loop = None if self._proc is not None: self._proc.terminate() -if loop is None: -# In this case spawn internals would have used -# portage's global loop when attaching a waiter to -# self._proc, so we are obligated to use that. -global_event_loop().run_until_complete(self._proc.wait()) -else: -if self._proc_waiter is None: -self._proc_waiter = asyncio.ensure_future( -self._proc.wait(), loop=loop -) -future = asyncio.shield(self._proc_waiter) - -if loop is not None and future is None: -future = loop.create_future() -future.set_result(None) +if self._proc_waiter is None: +self._proc_waiter
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/, ...
commit: 6ce2be8d454f95c508d9f547d13487f9de863bdd
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 3 19:53:11 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 3 19:53:11 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6ce2be8d
_validate_deps: Discard configdict["pkg"]["USE"]
Since configdict["pkg"]["USE"] may contain package.use settings
from config.setcpv, it is inappropriate to use here (bug 675748),
so discard it. This is only an issue because configdict["pkg"] is
a sub-optimal place to extract metadata from. This issue does not
necessarily indicate a flaw in the Package constructor, since
passing in precalculated USE can be valid for things like
autounmask USE changes.
Bug: https://bugs.gentoo.org/675748
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/doebuild.py | 12
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 19 ++-
lib/portage/tests/resolver/ResolverPlayground.py | 1 +
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 942fa90101..6691db4e97 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -1813,6 +1813,14 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
invalid_dep_exempt_phases = {"clean", "cleanrm", "help", "prerm", "postrm"}
all_keys = set(Package.metadata_keys)
all_keys.add("SRC_URI")
+# Since configdict["pkg"]["USE"] may contain package.use settings
+# from config.setcpv, it is inappropriate to use here (bug 675748),
+# so discard it. This is only an issue because configdict["pkg"] is
+# a sub-optimal place to extract metadata from. This issue does not
+# necessarily indicate a flaw in the Package constructor, since
+# passing in precalculated USE can be valid for things like
+# autounmask USE changes.
+all_keys.discard("USE")
all_keys = tuple(all_keys)
metadata = mysettings.configdict["pkg"]
if all(k in metadata for k in ("PORTAGE_REPO_NAME", "SRC_URI")):
@@ -1838,6 +1846,10 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
root_config = RootConfig(mysettings, {"porttree": FakeTree(mydbapi)}, None)
+# A USE calculation from setcpv should always be available here because
+# mysettings.mycpv is not None, so use it to prevent redundant setcpv
calls.
+metadata["USE"] = mysettings["PORTAGE_USE"]
+
pkg = Package(
built=False,
cpv=mysettings.mycpv,
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index b38605bb90..445fcf6c4e 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -51,10 +51,23 @@ class DoebuildFdPipesTestCase(TestCase):
ebuilds = {
"app-misct/foo-1": {
"EAPI": "8",
+"IUSE": "+foo +bar",
+"REQUIRED_USE": "|| ( foo bar )",
"MISC_CONTENT": ebuild_body,
}
}
+# Populate configdict["pkg"]["USE"] with something arbitrary in order
+# to try and trigger bug 675748 in doebuild _validate_deps.
+arbitrary_package_use = "baz"
+
+user_config = {
+# In order to trigger bug 675748, package.env must be non-empty,
+# but the referenced env file can be empty.
+"package.env": (f"app-misct/foo {os.devnull}",),
+"package.use": (f"app-misct/foo {arbitrary_package_use}",),
+}
+
# Override things that may be unavailable, or may have portability
# issues when running tests in exotic environments.
# prepstrip - bug #447810 (bash read builtin EINTR problem)
@@ -63,7 +76,7 @@ class DoebuildFdPipesTestCase(TestCase):
self.assertEqual(true_binary is None, False, "true command not found")
dev_null = open(os.devnull, "wb")
-playground = ResolverPlayground(ebuilds=ebuilds)
+playground = ResolverPlayground(ebuilds=ebuilds,
user_config=user_config)
try:
QueryCommand._db = playground.trees
root_config = playground.trees[playground.eroot]["root_config"]
@@ -106,6 +119,10 @@ class DoebuildFdPipesTestCase(TestCase):
)
settings.setcpv(pkg)
+# Demonstrate that settings.configdict["pkg"]["USE"] contains our
arbitrary
+# package.use setting in order to trigger bug 675748.
+self.assertEqual(settings.configdict["pkg"]["USE"],
arbitrary_package_use)
+
# Try to trigger the config.environ() split_LC_ALL assertion for
bug 925863.
settings["LC_ALL"] = "C"
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index c0455415a1..f52a98f8db
[gentoo-commits] proj/portage:master commit in: lib/portage/elog/, lib/portage/, lib/portage/util/_eventloop/
commit: c3ebdbb42e72335ca65335c855a82b99537c7606
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 3 06:30:50 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 3 06:30:50 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c3ebdbb4
elog/mod_custom: Spawn processes in background
Since elog_process is typically called while the event loop is
running, hold references to spawned processes and wait for them
asynchronously, ultimately waiting for them if necessary when
the AsyncioEventLoop _close_main method calls _async_finalize
via portage.process.run_coroutine_exitfuncs().
ConfigProtectTestCase is useful for exercising this code, and
this little make.globals patch can be used to test failure
during finalize with this error message:
!!! PORTAGE_ELOG_COMMAND failed with exitcode 1
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -144 +144,2 @@ PORTAGE_ELOG_CLASSES="log warn error"
-PORTAGE_ELOG_SYSTEM="save_summary:log,warn,error,qa echo"
+PORTAGE_ELOG_SYSTEM="save_summary:log,warn,error,qa echo custom"
+PORTAGE_ELOG_COMMAND="/bin/false"
Bug: https://bugs.gentoo.org/925907
Signed-off-by: Zac Medico gentoo.org>
lib/portage/elog/mod_custom.py| 73 +--
lib/portage/process.py| 29 +
lib/portage/util/_eventloop/asyncio_event_loop.py | 8 ++-
3 files changed, 105 insertions(+), 5 deletions(-)
diff --git a/lib/portage/elog/mod_custom.py b/lib/portage/elog/mod_custom.py
index e0ae77e100..a3e199bcb7 100644
--- a/lib/portage/elog/mod_custom.py
+++ b/lib/portage/elog/mod_custom.py
@@ -1,10 +1,33 @@
# elog/mod_custom.py - elog dispatch module
-# Copyright 2006-2020 Gentoo Authors
+# Copyright 2006-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import types
+
+import portage
import portage.elog.mod_save
import portage.exception
import portage.process
+from portage.util.futures import asyncio
+
+# Since elog_process is typically called while the event loop is
+# running, hold references to spawned processes and wait for them
+# asynchronously, ultimately waiting for them if necessary when
+# the AsyncioEventLoop _close_main method calls _async_finalize
+# via portage.process.run_coroutine_exitfuncs().
+_proc_refs = None
+
+
+def _get_procs() -> list[tuple[portage.process.MultiprocessingProcess,
asyncio.Future]]:
+"""
+Return list of (proc, asyncio.ensure_future(proc.wait())) which is not
+inherited from the parent after fork.
+"""
+global _proc_refs
+if _proc_refs is None or _proc_refs.pid != portage.getpid():
+_proc_refs = types.SimpleNamespace(pid=portage.getpid(), procs=[])
+portage.process.atexit_register(_async_finalize)
+return _proc_refs.procs
def process(mysettings, key, logentries, fulltext):
@@ -18,8 +41,50 @@ def process(mysettings, key, logentries, fulltext):
mylogcmd = mysettings["PORTAGE_ELOG_COMMAND"]
mylogcmd = mylogcmd.replace("${LOGFILE}", elogfilename)
mylogcmd = mylogcmd.replace("${PACKAGE}", key)
-retval = portage.process.spawn_bash(mylogcmd)
-if retval != 0:
+loop = asyncio.get_event_loop()
+proc = portage.process.spawn_bash(mylogcmd, returnproc=True)
+procs = _get_procs()
+procs.append((proc, asyncio.ensure_future(proc.wait(), loop=loop)))
+for index, (proc, waiter) in reversed(list(enumerate(procs))):
+if not waiter.done():
+continue
+del procs[index]
+if waiter.result() != 0:
+raise portage.exception.PortageException(
+f"!!! PORTAGE_ELOG_COMMAND failed with exitcode
{waiter.result()}"
+)
+
+
+async def _async_finalize():
+"""
+Async finalize is preferred, since we can wait for process exit status.
+"""
+procs = _get_procs()
+while procs:
+proc, waiter = procs.pop()
+if (await waiter) != 0:
+raise portage.exception.PortageException(
+f"!!! PORTAGE_ELOG_COMMAND failed with exitcode
{waiter.result()}"
+)
+
+
+def finalize():
+"""
+NOTE: This raises PortageException if there are any processes
+still running, so it's better to use _async_finalize instead
+(invoked via portage.process.run_coroutine_exitfuncs() in
+the AsyncioEventLoop _close_main method).
+"""
+procs = _get_procs()
+while procs:
+proc, waiter = procs.pop()
+if not waiter.done():
+waiter.cancel()
+proc.terminate()
+raise portage.exception.PortageException(
+f"!!! PORTAGE_ELOG_COMMAND was killed after it was found
running in the background (pid {proc.pid})"
+)
+elif waiter.result() != 0:
raise portage.exception.PortageException(
-"!!! PORTAGE_ELOG_COMMAND failed with exitcode %d" % retval
+
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 62ee9bf8c680b2a18713da5bd453e3a771257409
Author: Zac Medico gentoo org>
AuthorDate: Sat Mar 2 22:48:54 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Mar 2 22:50:54 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=62ee9bf8
binarytree._populate_remote: Fix UnboundLocalError for binpkg-request-signature
If an InvalidBinaryPackageFormat exception was raised from
get_binpkg_format for binpkg-request-signature then it
triggered an UnboundLocalError here.
Fixes: 445f10f4214c ("Use binpkg extensions and header to get format")
Bug: https://bugs.gentoo.org/926048
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dbapi/bintree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index f4251b47d6..4ba1407cda 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1624,7 +1624,7 @@ class binarytree:
binpkg_format = get_binpkg_format(
d.get("PATH"), remote=True
)
-except InvalidBinaryPackageFormat:
+except InvalidBinaryPackageFormat as e:
writemsg(
colorize(
"WARN",
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
commit: fe510e099bc9a8055c3ee50fced47fc3dc7ba166
Author: Zac Medico gentoo org>
AuthorDate: Fri Mar 1 17:09:56 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Mar 1 18:08:51 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fe510e09
doebuild: Call _setup_locale
Call _setup_locale in order to prevent an AssertionError from
config.environ() for the config phase (or any other phase for
that matter).
For returnproc or returnpid assume that the event loop is running
so we can't run the event loop to call _setup_locale in this case
and we have to assume the caller took care of it (otherwise
config.environ() will raise AssertionError).
Update DoebuildFdPipesTestCase to use EAPI 8 and test the
pkg_config function with an ebuild located in /var/db/pkg just
like emerge --config does. Set LC_ALL=C just before doebuild
calls in order to try and trigger the config.environ()
split_LC_ALL assertion.
Bug: https://bugs.gentoo.org/925863
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/doebuild.py | 8 +++
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 77 --
2 files changed, 52 insertions(+), 33 deletions(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index bc51fdff2d..942fa90101 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -43,6 +43,7 @@ portage.proxy.lazyimport.lazyimport(
"portage.util._async.SchedulerInterface:SchedulerInterface",
"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util.ExtractKernelVersion:ExtractKernelVersion",
+"_emerge.EbuildPhase:_setup_locale",
)
from portage import (
@@ -1034,6 +1035,13 @@ def doebuild(
myebuild, mydo, myroot, mysettings, debug, use_cache, mydbapi
)
+# For returnproc or returnpid assume that the event loop is running
+# so we can't run the event loop to call _setup_locale in this case
+# and we have to assume the caller took care of it (otherwise
+# config.environ() will raise AssertionError).
+if not (returnproc or returnpid):
+asyncio.run(_setup_locale(mysettings))
+
if mydo in clean_phases:
builddir_lock = None
if not returnpid and "PORTAGE_BUILDDIR_LOCKED" not in mysettings:
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index 678486ed16..b38605bb90 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Gentoo Authors
+# Copyright 2013-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import multiprocessing
@@ -27,20 +27,22 @@ class DoebuildFdPipesTestCase(TestCase):
output_fd = self.output_fd
ebuild_body = ["S=${WORKDIR}"]
-for phase_func in (
-"pkg_info",
-"pkg_nofetch",
-"pkg_pretend",
-"pkg_setup",
-"src_unpack",
-"src_prepare",
-"src_configure",
-"src_compile",
-"src_test",
-"src_install",
+for phase_func, default in (
+("pkg_info", False),
+("pkg_nofetch", False),
+("pkg_pretend", False),
+("pkg_setup", False),
+("pkg_config", False),
+("src_unpack", False),
+("src_prepare", True),
+("src_configure", False),
+("src_compile", False),
+("src_test", False),
+("src_install", False),
):
ebuild_body.append(
-("%s() { echo ${EBUILD_PHASE}" " 1>&%s; }") % (phase_func,
output_fd)
+("%s() { %secho ${EBUILD_PHASE}" " 1>&%s; }")
+% (phase_func, "default; " if default else "", output_fd)
)
ebuild_body.append("")
@@ -48,7 +50,7 @@ class DoebuildFdPipesTestCase(TestCase):
ebuilds = {
"app-misct/foo-1": {
-"EAPI": "5",
+"EAPI": "8",
"MISC_CONTENT": ebuild_body,
}
}
@@ -103,24 +105,33 @@ class DoebuildFdPipesTestCase(TestCase):
type_name="ebuild",
)
settings.setcpv(pkg)
-ebuildpath = portdb.findname(cpv)
-self.assertNotEqual(ebuildpath, None)
-
-for phase in (
-"info",
-"nofetch",
-"pretend",
-"setup",
-"unpack",
-"prepare",
-"configure",
-"compile",
-"test",
-"install",
-"qmerge",
-"clean",
-"merge",
+
+# Try to trigger the
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: d8089d1af39c80e1edfb1669ae92fef7ab30e08a
Author: Mike Gilbert gentoo org>
AuthorDate: Fri Mar 1 15:49:26 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 1 16:19:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8089d1a
Improve whitespace handling when parsing /proc/self/mountinfo
Only break lines on "\n" (line feed).
Only split lines on " " (space).
Bug: https://bugs.gentoo.org/925888
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/writeable_check.py | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/portage/util/writeable_check.py
b/lib/portage/util/writeable_check.py
index 3427315cf7..ad1d9edff0 100644
--- a/lib/portage/util/writeable_check.py
+++ b/lib/portage/util/writeable_check.py
@@ -47,6 +47,7 @@ def linux_ro_checker(dir_list):
"/proc/self/mountinfo",
encoding=_encodings["content"],
errors="replace",
+newline="\n",
) as f:
for line in f:
# we're interested in dir and both attr fields which always
@@ -58,7 +59,7 @@ def linux_ro_checker(dir_list):
# to the left of the ' - ', after the attr's, so split it there
mount = line.split(" - ", 1)
try:
-_dir, attr1 = mount[0].split()[4:6]
+_dir, attr1 = mount[0].split(" ")[4:6]
except ValueError:
# If it raises ValueError we can simply ignore the line.
invalids.append(line)
@@ -68,10 +69,10 @@ def linux_ro_checker(dir_list):
# for example: 16 1 0:16 / /root rw,noatime - lxfs rw
if len(mount) > 1:
try:
-attr2 = mount[1].split()[2]
+attr2 = mount[1].split(" ")[2]
except IndexError:
try:
-attr2 = mount[1].split()[1]
+attr2 = mount[1].split(" ")[1]
except IndexError:
invalids.append(line)
continue
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, src/
commit: a860484b6e8c8d107255f2105796ae7f58812e9f
Author: Mike Gilbert gentoo org>
AuthorDate: Thu Feb 29 19:38:20 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Thu Feb 29 19:38:20 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a860484b
Drop portage.util.libc extension module
This was originally added as a workaround for musl, where
ctypes.util.find_library fails.
Instead, we now try to load "libc.so" as a fallback and can just rely on
ctypes to call tolower() and toupper().
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/locale.py | 9 +++
src/meson.build| 9 ---
src/portage_util_libc.c| 67 --
3 files changed, 3 insertions(+), 82 deletions(-)
diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py
index d0edeb4afe..f45d761760 100644
--- a/lib/portage/util/locale.py
+++ b/lib/portage/util/locale.py
@@ -43,12 +43,9 @@ def _check_locale(silent):
"""
The inner locale check function.
"""
-try:
-from portage.util import libc
-except ImportError:
-(libc, _) = load_libc()
-if libc is None:
-return None
+(libc, _) = load_libc()
+if libc is None:
+return None
lc = list(range(ord("a"), ord("z") + 1))
uc = list(range(ord("A"), ord("Z") + 1))
diff --git a/src/meson.build b/src/meson.build
index cbc7aa611c..6a36724ceb 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -2,14 +2,6 @@
# and for development. Meson does not allow you to build in-place and Python
# cannot create a single namespace from two identically-named paths.
-libc_ext = py.extension_module(
-'libc',
-'portage_util_libc.c',
-dependencies : py.dependency(),
-subdir : 'portage' / 'util',
-install : true
-)
-
whirlpool_ext = py.extension_module(
'_whirlpool',
'portage_util__whirlpool.c',
@@ -21,7 +13,6 @@ whirlpool_ext = py.extension_module(
run_command(
[
'ln', '-srnf',
-libc_ext.full_path(),
whirlpool_ext.full_path(),
meson.project_source_root() / 'lib' / 'portage' / 'util/'
],
diff --git a/src/portage_util_libc.c b/src/portage_util_libc.c
deleted file mode 100644
index 12b2440c72..00
--- a/src/portage_util_libc.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/* Copyright 2005-2020 Gentoo Authors
- * Distributed under the terms of the GNU General Public License v2
- */
-
-#include
-#include
-#include
-
-static PyObject * _libc_tolower(PyObject *, PyObject *);
-static PyObject * _libc_toupper(PyObject *, PyObject *);
-
-static PyMethodDef LibcMethods[] = {
- {
- .ml_name = "tolower",
- .ml_meth = _libc_tolower,
- .ml_flags = METH_VARARGS,
- .ml_doc = "Convert to lower case using system locale."
-
- },
- {
- .ml_name = "toupper",
- .ml_meth = _libc_toupper,
- .ml_flags = METH_VARARGS,
- .ml_doc = "Convert to upper case using system locale."
- },
- {NULL, NULL, 0, NULL}
-};
-
-static struct PyModuleDef moduledef = {
- PyModuleDef_HEAD_INIT,
- .m_name = "libc",
- .m_doc = "Module for converting case using the system locale",
- .m_size = -1,
- .m_methods = LibcMethods,
-};
-
-PyMODINIT_FUNC
-PyInit_libc(void)
-{
- PyObject *m;
- m = PyModule_Create();
- return m;
-}
-
-
-static PyObject *
-_libc_tolower(PyObject *self, PyObject *args)
-{
- int c;
-
- if (!PyArg_ParseTuple(args, "i", ))
- return NULL;
-
- return Py_BuildValue("i", tolower(c));
-}
-
-
-static PyObject *
-_libc_toupper(PyObject *self, PyObject *args)
-{
- int c;
-
- if (!PyArg_ParseTuple(args, "i", ))
- return NULL;
-
- return Py_BuildValue("i", toupper(c));
-}
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/, ...
commit: b2d8226af4589d95f44d20d441056f645a523039
Author: Zac Medico gentoo org>
AuthorDate: Thu Feb 29 04:26:02 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Feb 29 04:37:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2d8226a
Delete compat_coroutine module
The compat_coroutine module has been unused since the migration
to PEP 492 async and await syntax in 2021, which began in commit
b3b9acc13c43 and was completed in commit bcda30d0a6fa.
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/futures/meson.build | 1 -
.../tests/util/futures/test_compat_coroutine.py| 210 -
lib/portage/util/futures/_asyncio/__init__.py | 15 +-
lib/portage/util/futures/compat_coroutine.py | 142 --
lib/portage/util/futures/meson.build | 1 -
5 files changed, 1 insertion(+), 368 deletions(-)
diff --git a/lib/portage/tests/util/futures/meson.build
b/lib/portage/tests/util/futures/meson.build
index 877acc27cd..cb78314844 100644
--- a/lib/portage/tests/util/futures/meson.build
+++ b/lib/portage/tests/util/futures/meson.build
@@ -1,6 +1,5 @@
py.install_sources(
[
-'test_compat_coroutine.py',
'test_done_callback.py',
'test_done_callback_after_exit.py',
'test_iter_completed.py',
diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py
b/lib/portage/tests/util/futures/test_compat_coroutine.py
deleted file mode 100644
index b25708886c..00
--- a/lib/portage/tests/util/futures/test_compat_coroutine.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Copyright 2018 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import (
-coroutine,
-coroutine_return,
-)
-from portage.util.futures._sync_decorator import _sync_decorator, _sync_methods
-from portage.tests import TestCase
-
-
-class CompatCoroutineTestCase(TestCase):
-def test_returning_coroutine(self):
-@coroutine
-def returning_coroutine(loop=None):
-yield asyncio.sleep(0, loop=loop)
-coroutine_return("success")
-
-loop = asyncio.get_event_loop()
-self.assertEqual(
-"success",
-
asyncio.get_event_loop().run_until_complete(returning_coroutine(loop=loop)),
-)
-
-def test_raising_coroutine(self):
-class TestException(Exception):
-pass
-
-@coroutine
-def raising_coroutine(loop=None):
-yield asyncio.sleep(0, loop=loop)
-raise TestException("exception")
-
-loop = asyncio.get_event_loop()
-self.assertRaises(
-TestException, loop.run_until_complete,
raising_coroutine(loop=loop)
-)
-
-def test_catching_coroutine(self):
-class TestException(Exception):
-pass
-
-@coroutine
-def catching_coroutine(loop=None):
-loop = asyncio._wrap_loop(loop)
-future = loop.create_future()
-loop.call_soon(future.set_exception, TestException("exception"))
-try:
-yield future
-except TestException:
-self.assertTrue(True)
-else:
-self.assertTrue(False)
-coroutine_return("success")
-
-loop = asyncio.get_event_loop()
-self.assertEqual(
-"success", loop.run_until_complete(catching_coroutine(loop=loop))
-)
-
-def test_cancelled_coroutine(self):
-"""
-Verify that a coroutine can handle (and reraise) asyncio.CancelledError
-in order to perform any necessary cleanup. Note that the
-asyncio.CancelledError will only be thrown in the coroutine if there's
-an opportunity (yield) before the generator raises StopIteration.
-"""
-loop = asyncio.get_event_loop()
-ready_for_exception = loop.create_future()
-exception_in_coroutine = loop.create_future()
-
-@coroutine
-def cancelled_coroutine(loop=None):
-loop = asyncio._wrap_loop(loop)
-while True:
-task = loop.create_future()
-try:
-ready_for_exception.set_result(None)
-yield task
-except BaseException as e:
-# Since python3.8, asyncio.CancelledError inherits
-# from BaseException.
-task.done() or task.cancel()
-exception_in_coroutine.set_exception(e)
-raise
-else:
-exception_in_coroutine.set_result(None)
-
-future = cancelled_coroutine(loop=loop)
-loop.run_until_complete(ready_for_exception)
-future.cancel()
-
-self.assertRaises(asyncio.CancelledError, loop.run_until_complete,
future)
-
-
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: fa7fb4c5119aa077a0731e1f8892ea0791a4968b
Author: Sam James gentoo org>
AuthorDate: Wed Feb 28 16:00:57 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 28 16:01:04 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa7fb4c5
gpkg: placate black
Fixes: 957902f84edece635210689f46e20741e76d9dba
Signed-off-by: Sam James gentoo.org>
lib/portage/gpkg.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 5cd1f2394e..2b957d58c4 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -656,7 +656,9 @@ class tar_safe_extract:
raise ValueError("Path traversal detected.")
if member.isdev():
writemsg(
-colorize("BAD", f"Danger: device file detected:
{member.name}\n")
+colorize(
+"BAD", f"Danger: device file detected:
{member.name}\n"
+)
)
raise ValueError("Device file detected.")
if member.islnk() and (member.linkname not in self.file_list):
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 957902f84edece635210689f46e20741e76d9dba
Author: Sam James gentoo org>
AuthorDate: Wed Feb 28 15:51:15 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 28 15:51:15 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=957902f8
gpkg: add missing new lines to error messages
Signed-off-by: Sam James gentoo.org>
lib/portage/gpkg.py | 18 +-
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index edb0e43fbf..5cd1f2394e 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -382,7 +382,7 @@ class tar_stream_reader:
try:
if self.proc.wait() != os.EX_OK:
if not self.killed:
-writemsg(colorize("BAD", f"GPKG external program
failed."))
+writemsg(colorize("BAD", f"GPKG external program
failed.\n"))
raise CompressorOperationFailed("decompression failed")
finally:
self.proc.stdout.close()
@@ -418,7 +418,7 @@ class checksum_helper:
else:
self.uid = pwd.getpwnam(drop_user).pw_uid
except KeyError:
-writemsg(colorize("BAD", f"!!! Failed to find user
{drop_user}."))
+writemsg(colorize("BAD", f"!!! Failed to find user
{drop_user}.\n"))
raise
try:
@@ -428,7 +428,7 @@ class checksum_helper:
else:
self.gid = grp.getgrnam(drop_group).gr_gid
except KeyError:
-writemsg(colorize("BAD", f"!!! Failed to find group
{drop_group}."))
+writemsg(colorize("BAD", f"!!! Failed to find group
{drop_group}.\n"))
raise
else:
self.uid = None
@@ -636,33 +636,33 @@ class tar_safe_extract:
):
writemsg(
colorize(
-"BAD", f"Danger: duplicate files detected:
{member.name}"
+"BAD", f"Danger: duplicate files detected:
{member.name}\n"
)
)
raise ValueError("Duplicate files detected.")
if member.name.startswith("/"):
writemsg(
colorize(
-"BAD", f"Danger: absolute path detected:
{member.name}"
+"BAD", f"Danger: absolute path detected:
{member.name}\n"
)
)
raise ValueError("Absolute path detected.")
if member.name.startswith("../") or ("/../" in member.name):
writemsg(
colorize(
-"BAD", f"Danger: path traversal detected:
{member.name}"
+"BAD", f"Danger: path traversal detected:
{member.name}\n"
)
)
raise ValueError("Path traversal detected.")
if member.isdev():
writemsg(
-colorize("BAD", f"Danger: device file detected:
{member.name}")
+colorize("BAD", f"Danger: device file detected:
{member.name}\n")
)
raise ValueError("Device file detected.")
if member.islnk() and (member.linkname not in self.file_list):
writemsg(
colorize(
-"BAD", f"Danger: hardlink escape detected:
{member.name}"
+"BAD", f"Danger: hardlink escape detected:
{member.name}\n"
)
)
raise ValueError("Hardlink escape detected.")
@@ -995,7 +995,7 @@ class gpkg:
image_safe.extractall(decompress_dir)
image_tar.close()
except Exception as ex:
-writemsg(colorize("BAD", "!!!Extract failed."))
+writemsg(colorize("BAD", "!!!Extract failed.\n"))
raise
finally:
if not image_tar.closed:
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: f070275fe05d5053c3756ebb5d0a602db8ba515d
Author: Sam James gentoo org>
AuthorDate: Wed Feb 28 15:48:48 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 28 15:49:16 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f070275f
binpkg: add another missing newline to error message
```
Error reading binpkg
'/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar': [Errno 22]
Invalid argument!!! Invalid binary package:
'/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar', Error reading
binpkg '/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar': [Errno
22] Invalid argument
```
Bug: https://bugs.gentoo.org/925714
Signed-off-by: Sam James gentoo.org>
lib/portage/binpkg.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
index 9ecd52cf3c..a48e09bdb8 100644
--- a/lib/portage/binpkg.py
+++ b/lib/portage/binpkg.py
@@ -54,7 +54,7 @@ def get_binpkg_format(binpkg_path, check_file=False,
remote=False):
# We got many different exceptions here, so have to catch all of them.
file_format = None
writemsg(
-colorize("ERR", f"Error reading binpkg '{binpkg_path}': {err}"),
+colorize("ERR", f"Error reading binpkg '{binpkg_path}': {err}\n"),
)
raise InvalidBinaryPackageFormat(f"Error reading binpkg
'{binpkg_path}': {err}")
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/
commit: e882b1e956d50808a0143875a8ca35f2fc21f368
Author: Zac Medico gentoo org>
AuthorDate: Wed Feb 28 06:25:45 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 28 06:25:45 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e882b1e9
UnshareNetTestCase: Initialize ABILITY_TO_UNSHARE in setUp
Initialize ABILITY_TO_UNSHARE in setUp so that _unshare_validate
uses the correct PORTAGE_MULTIPROCESSING_START_METHOD setup
from super().setUp(), eliminating messages like this from CI
runs for the multiprocessing start method spawn:
/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/popen_fork.py:66:
DeprecationWarning: This process (pid=2886) is multi-threaded, use of
fork() may lead to deadlocks in the child.
The cause of these messages can be traced by patching python's
multiprocessing popen_fork.py like this:
--- /usr/lib/python3.12/multiprocessing/popen_fork.py
+++ /usr/lib/python3.12/multiprocessing/popen_fork.py
@@ -2,2 +2,3 @@
import signal
+import traceback
@@ -62,2 +63,3 @@
def _launch(self, process_obj):
+traceback.print_stack()
code = 1
Fixes: 3110ec376cbc ("actions: Fix interaction between start-method and
pytest-xdist")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/process/test_unshare_net.py | 18 +-
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/lib/portage/tests/process/test_unshare_net.py
b/lib/portage/tests/process/test_unshare_net.py
index dabf15585f..ad3b288ef4 100644
--- a/lib/portage/tests/process/test_unshare_net.py
+++ b/lib/portage/tests/process/test_unshare_net.py
@@ -20,19 +20,27 @@ ping -c 1 -W 1 10.0.0.1 || exit 1
ping -c 1 -W 1 ::1 || exit 1
ping -c 1 -W 1 fd::1 || exit 1
"""
-ABILITY_TO_UNSHARE = portage.process._unshare_validate(CLONE_NEWNET)
class UnshareNetTestCase(TestCase):
-@pytest.mark.skipif(
-ABILITY_TO_UNSHARE != 0,
-reason=f"Unable to unshare: {errno.errorcode.get(ABILITY_TO_UNSHARE,
'?')}",
-)
+def setUp(self):
+"""
+Initialize ABILITY_TO_UNSHARE in setUp so that _unshare_validate
+uses the correct PORTAGE_MULTIPROCESSING_START_METHOD setup
+from super().setUp().
+"""
+super().setUp()
+self.ABILITY_TO_UNSHARE =
portage.process._unshare_validate(CLONE_NEWNET)
+
@pytest.mark.skipif(
portage.process.find_binary("ping") is None, reason="ping not found"
)
@pytest.mark.skipif(platform.system() != "Linux", reason="not Linux")
def testUnshareNet(self):
+if self.ABILITY_TO_UNSHARE != 0:
+pytest.skip(
+f"Unable to unshare:
{errno.errorcode.get(self.ABILITY_TO_UNSHARE, '?')}"
+)
env = os.environ.copy()
env["IPV6"] = "1" if portage.process._has_ipv6() else ""
self.assertEqual(
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dbapi/
commit: e9bdb7342b3048ab3236bff9d94ce733bb877d8e
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 27 04:02:28 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 27 04:03:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e9bdb734
BinarytreeTestCase: Use temporary TMPDIR
Create a temporary TMPDIR which prevents test methods of this
class from leaving behind an empty /tmp/Packages file if TMPDIR
is initially unset.
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/dbapi/test_bintree.py | 22 +-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/lib/portage/tests/dbapi/test_bintree.py
b/lib/portage/tests/dbapi/test_bintree.py
index 018f1cf9bd..91ac338a05 100644
--- a/lib/portage/tests/dbapi/test_bintree.py
+++ b/lib/portage/tests/dbapi/test_bintree.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Gentoo Authors
+# Copyright 2022-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from unittest.mock import MagicMock, patch, call
@@ -13,6 +13,26 @@ from portage.const import BINREPOS_CONF_FILE
class BinarytreeTestCase(TestCase):
+@classmethod
+def setUpClass(cls):
+"""
+Create a temporary TMPDIR which prevents test
+methods of this class from leaving behind an empty
+/tmp/Packages file if TMPDIR is initially unset.
+"""
+cls._orig_tmpdir = os.environ.get("TMPDIR")
+cls._tmpdir = tempfile.TemporaryDirectory()
+os.environ["TMPDIR"] = cls._tmpdir.name
+
+@classmethod
+def tearDownClass(cls):
+cls._tmpdir.cleanup()
+if cls._orig_tmpdir is None:
+os.environ.pop("TMPDIR", None)
+else:
+os.environ["TMPDIR"] = cls._orig_tmpdir
+del cls._orig_tmpdir, cls._tmpdir
+
def test_required_init_params(self):
with self.assertRaises(TypeError) as cm:
binarytree()
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/resolver/, cnf/sets/, lib/portage/tests/sets/base/
commit: 1d3e3843f2a51c581d344540c5c6ee266afa30d2
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 25 22:57:43 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 26 23:17:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1d3e3843
cnf: sets: Migrate @golang-rebuild to dev-lang/go
Bug: https://bugs.gentoo.org/919751
Signed-off-by: Zac Medico gentoo.org>
cnf/sets/portage.conf| 6 --
lib/portage/tests/resolver/ResolverPlayground.py | 10 +-
lib/portage/tests/sets/base/test_variable_set.py | 8 ++--
3 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf
index 2e02f91f97..c272f98db1 100644
--- a/cnf/sets/portage.conf
+++ b/cnf/sets/portage.conf
@@ -110,12 +110,6 @@ class = portage.sets.dbapi.UnavailableBinaries
[changed-deps]
class = portage.sets.dbapi.ChangedDepsSet
-# Installed packages for which vdb *DEPEND includes dev-lang/go.
-[golang-rebuild]
-class = portage.sets.dbapi.VariableSet
-variable = BDEPEND
-includes = dev-lang/go
-
# Installed packages for which vdb *DEPEND includes virtual/rust
[rust-rebuild]
class = portage.sets.dbapi.VariableSet
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index 75c86b615c..c0455415a1 100644
--- a/lib/portage/tests/resolver/ResolverPlayground.py
+++ b/lib/portage/tests/resolver/ResolverPlayground.py
@@ -28,7 +28,7 @@ from portage.exception import InvalidBinaryPackageFormat
from portage.gpg import GPG
import _emerge
-from _emerge.actions import _calc_depclean
+from _emerge.actions import _calc_depclean, expand_set_arguments
from _emerge.Blocker import Blocker
from _emerge.create_depgraph_params import create_depgraph_params
from _emerge.DependencyArg import DependencyArg
@@ -747,6 +747,14 @@ class ResolverPlayground:
self.settings, self.trees, options, params, None
)
+atoms, retval = expand_set_arguments(
+atoms, action, self.trees[self.eroot]["root_config"]
+)
+if retval != os.EX_OK:
+raise AssertionError(
+f"expand_set_arguments failed with retval {retval}"
+)
+
if params_action == "remove":
depclean_result = _calc_depclean(
self.settings,
diff --git a/lib/portage/tests/sets/base/test_variable_set.py
b/lib/portage/tests/sets/base/test_variable_set.py
index 9e90ee6dd7..60c43a5b83 100644
--- a/lib/portage/tests/sets/base/test_variable_set.py
+++ b/lib/portage/tests/sets/base/test_variable_set.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Gentoo Authors
+# Copyright 2022-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -10,6 +10,10 @@ from portage.tests.resolver.ResolverPlayground import (
class VariableSetTestCase(TestCase):
def testVariableSetEmerge(self):
+
+# Using local set definition because @golang-rebuild migrated to
dev-lang/go since bug 919751.
+golang_rebuild =
"{class=portage.sets.dbapi.VariableSet,variable=BDEPEND,includes=dev-lang/go}"
+
ebuilds = {
"dev-go/go-pkg-1": {"BDEPEND": "dev-lang/go"},
"www-client/firefox-1": {
@@ -21,7 +25,7 @@ class VariableSetTestCase(TestCase):
test_cases = (
ResolverPlaygroundTestCase(
-["@golang-rebuild"],
+[f"@golang-rebuild{golang_rebuild}"],
mergelist=["dev-go/go-pkg-1"],
success=True,
),
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/, lib/_emerge/
commit: 0ff7a3b28e0ec63d68d32e01145db8962d53774d
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 26 05:09:21 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 26 05:09:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0ff7a3b2
SpawnProcess: Wait for async set_term_size
Use the SpawnProcess _unregister method to handle async
set_term_size results, avoiding possible messages like
bug 925456 triggered:
[ERROR] Task was destroyed but it is pending!
Also update _BinpkgFetcherProcess and _EbuildFetcherProcess
which inherit the _pty_ready attribute from SpawnProcess.
Fixes: f97e414ce980 ("set_term_size: Wait asynchronously if event loop is
running")
Bug: https://bugs.gentoo.org/923750
Bug: https://bugs.gentoo.org/925456
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/BinpkgFetcher.py | 6 --
lib/_emerge/EbuildFetcher.py | 6 --
lib/_emerge/SpawnProcess.py | 14 +-
lib/portage/output.py| 13 +
lib/portage/util/_pty.py | 23 ---
5 files changed, 46 insertions(+), 16 deletions(-)
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index a1524dc009..587e4a57a3 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.AsynchronousLock import AsynchronousLock
@@ -233,7 +233,9 @@ class _BinpkgFetcherProcess(SpawnProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def sync_timestamp(self):
diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py
index 7a45d95172..81d4b1054b 100644
--- a/lib/_emerge/EbuildFetcher.py
+++ b/lib/_emerge/EbuildFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -394,7 +394,9 @@ class _EbuildFetcherProcess(ForkProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def _eerror(self, lines):
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index b63afae01c..9fc12c42e5 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -46,6 +46,7 @@ class SpawnProcess(SubProcess):
+ (
"_main_task",
"_main_task_cancel",
+"_pty_ready",
"_selinux_type",
)
)
@@ -193,6 +194,9 @@ class SpawnProcess(SubProcess):
self._main_task.add_done_callback(self._main_exit)
async def _main(self, build_logger, pipe_logger, loop=None):
+if isinstance(self._pty_ready, asyncio.Future):
+await self._pty_ready
+self._pty_ready = None
try:
if pipe_logger.poll() is None:
await pipe_logger.async_wait()
@@ -238,7 +242,9 @@ class SpawnProcess(SubProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def _spawn(
@@ -258,6 +264,12 @@ class SpawnProcess(SubProcess):
SubProcess._unregister(self)
if self._main_task is not None:
self._main_task.done() or self._main_task.cancel()
+if isinstance(self._pty_ready, asyncio.Future):
+(
+self._pty_ready.done()
+and (self._pty_ready.cancelled() or self._pty_ready.result()
or True)
+) or self._pty_ready.cancel()
+self._pty_ready = None
def _cancel(self):
if self._main_task is not None:
diff --git a/lib/portage/output.py b/lib/portage/output.py
index 7d3a6278f3..4408705c45 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -8,6 +8,7 @@ import itertools
import re
import subprocess
import sys
+from typing import Optional
import portage
@@ -554,10 +555,16 @@ def get_term_size(fd=None):
return (0, 0)
-def set_term_size(lines, columns, fd):
+def set_term_size(lines: int, columns: int, fd: int) ->
Optional[asyncio.Future]:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/, /, lib/portage/
commit: 19e27e0415fd321c39104f7d687bcdc4f4132e24
Author: Mike Gilbert gentoo org>
AuthorDate: Sun Feb 25 18:10:15 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Mon Feb 26 04:10:32 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=19e27e04
Add workaround for loading libc on musl
musl libc has no soname, which causes ctypes.util.find_library to fail.
As a fallback, try to load "libc.so".
Signed-off-by: Mike Gilbert gentoo.org>
NEWS| 7 +++
lib/portage/dbapi/_MergeProcess.py | 4 ++--
lib/portage/dbapi/_SyncfsProcess.py | 14 --
lib/portage/process.py | 16 +---
lib/portage/util/_ctypes.py | 15 +++
lib/portage/util/locale.py | 7 ++-
6 files changed, 35 insertions(+), 28 deletions(-)
diff --git a/NEWS b/NEWS
index eb84651b53..258d800373 100644
--- a/NEWS
+++ b/NEWS
@@ -6,8 +6,15 @@ Release notes take the form of the following optional
categories:
* Bug fixes
* Cleanups
+portage-3.0.64 (UNRELEASED)
+--
+
+Bug fixes:
+
+
portage-3.0.63 (2024-02-25)
--
+* ctypes: Add workaround for loading libc on musl
Bug fixes:
* emerge: Skip installed packages with emptytree in depgraph selection (bug
#651018).
diff --git a/lib/portage/dbapi/_MergeProcess.py
b/lib/portage/dbapi/_MergeProcess.py
index dd5ad71cf8..d9ab2b47aa 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -10,7 +10,7 @@ import fcntl
import portage
from portage import os, _unicode_decode
from portage.package.ebuild._ipc.QueryCommand import QueryCommand
-from portage.util._ctypes import find_library
+from portage.util._ctypes import load_libc
import portage.elog.messages
from portage.util._async.ForkProcess import ForkProcess
from portage.util import no_color
@@ -64,7 +64,7 @@ class MergeProcess(ForkProcess):
# process, so that it's only done once rather than
# for each child process.
if platform.system() == "Linux" and "merge-sync" in settings.features:
-find_library("c")
+load_libc()
# Inherit stdin by default, so that the pdb SIGUSR1
# handler is usable for the subprocess.
diff --git a/lib/portage/dbapi/_SyncfsProcess.py
b/lib/portage/dbapi/_SyncfsProcess.py
index ddc2240071..300ae53985 100644
--- a/lib/portage/dbapi/_SyncfsProcess.py
+++ b/lib/portage/dbapi/_SyncfsProcess.py
@@ -4,7 +4,7 @@
import functools
from portage import os
-from portage.util._ctypes import find_library, LoadLibrary
+from portage.util._ctypes import load_libc
from portage.util._async.ForkProcess import ForkProcess
@@ -24,15 +24,9 @@ class SyncfsProcess(ForkProcess):
@staticmethod
def _get_syncfs():
-filename = find_library("c")
-if filename is not None:
-library = LoadLibrary(filename)
-if library is not None:
-try:
-return library.syncfs
-except AttributeError:
-pass
-
+(libc, _) = load_libc()
+if libc is not None:
+return getattr(libc, "syncfs", None)
return None
@staticmethod
diff --git a/lib/portage/process.py b/lib/portage/process.py
index cc9ed7bf78..b652e32942 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -37,7 +37,7 @@ portage.proxy.lazyimport.lazyimport(
from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
from portage.exception import CommandNotFound
from portage.proxy.objectproxy import ObjectProxy
-from portage.util._ctypes import find_library, LoadLibrary, ctypes
+from portage.util._ctypes import load_libc, LoadLibrary, ctypes
try:
from portage.util.netlink import RtNetlink
@@ -960,11 +960,9 @@ def _exec(
have_unshare = False
libc = None
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
-filename = find_library("c")
-if filename is not None:
-libc = LoadLibrary(filename)
-if libc is not None:
-have_unshare = hasattr(libc, "unshare")
+(libc, _) = load_libc()
+if libc is not None:
+have_unshare = hasattr(libc, "unshare")
if not have_unshare:
# unshare() may not be supported by libc
@@ -1212,11 +1210,7 @@ class _unshare_validator:
"""
# This ctypes library lookup caches the result for use in the
# subprocess when the multiprocessing start method is fork.
-filename = find_library("c")
-if filename is None:
-return errno.ENOTSUP
-
-libc = LoadLibrary(filename)
+(libc, filename) = load_libc()
if libc is None:
return errno.ENOTSUP
diff --git a/lib/portage/util/_ctypes.py b/lib/portage/util/_ctypes.py
index e6d1e327cb..04e965ba92 100644
--- a/lib/portage/util/_ctypes.py
+++ b/lib/portage/util/_ctypes.py
@@
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 0855821572f32e81b031a250f7491f34a2fd4078
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 24 23:29:29 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Feb 25 08:24:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=08558215
dbapi: Fix TypeError when passing Exception to warnings.warn
If an Exception is passed as a message to warnings.warn then
it triggers this TypeError:
if metadata_updates:
try:
aux_update(cpv, metadata_updates)
except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
> warnings.warn(e)
E TypeError: expected string or bytes-like object, got
'CorruptionKeyError'
Fixes: fb1d0a22f657 ("dbapi: KeyError tolerance during package moves")
Bug: https://bugs.gentoo.org/922935
Signed-off-by: Zac Medico gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1282
Signed-off-by: Sam James gentoo.org>
lib/portage/dbapi/__init__.py | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 6f95b93a21..9105227c77 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -1,11 +1,12 @@
-# Copyright 1998-2023 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["dbapi"]
import functools
+import logging
import re
-import warnings
+import sys
from typing import Any, Dict, List, Optional, Tuple
from collections.abc import Sequence
@@ -429,7 +430,9 @@ class dbapi:
try:
aux_update(cpv, metadata_updates)
except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
-warnings.warn(e)
+logging.warning(
+f"{e.__class__.__name__}: {e}", exc_info=sys.exc_info()
+)
if onUpdate:
onUpdate(maxval, i + 1)
if onProgress:
@@ -477,6 +480,6 @@ class dbapi:
try:
self.aux_update(mycpv, mydata)
except CorruptionKeyError as e:
-warnings.warn(e)
+logging.warning(f"{e.__class__.__name__}: {e}",
exc_info=sys.exc_info())
continue
return moves
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/emerge/
commit: 8a2f1d14788d107ec54dc53c9ef1cf00ee310d51 Author: Gábor Oszkár Dénes protonmail com> AuthorDate: Sat Feb 24 20:48:05 2024 + Commit: Sam James gentoo org> CommitDate: Sun Feb 25 08:25:06 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8a2f1d14 test_baseline: Improve robustness with cleanup The baseline tests need to cleanup the ResolverPlayground after each testcase, with each different parametrization. This is ensured by making the scope of the playground fixture the function instead of the module. With module the cleanup only happens before/after the switch from/to xpak and gpkg. Signed-off-by: Gábor Oszkár Dénes protonmail.com> Closes: https://github.com/gentoo/portage/pull/1281 Signed-off-by: Sam James gentoo.org> lib/portage/tests/emerge/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/tests/emerge/conftest.py b/lib/portage/tests/emerge/conftest.py index 580d1e09ab..356e09879c 100644 --- a/lib/portage/tests/emerge/conftest.py +++ b/lib/portage/tests/emerge/conftest.py @@ -406,7 +406,7 @@ def async_loop(): yield asyncio._wrap_loop() -@pytest.fixture(params=SUPPORTED_GENTOO_BINPKG_FORMATS, scope="module") +@pytest.fixture(params=SUPPORTED_GENTOO_BINPKG_FORMATS, scope="function") def playground(request, tmp_path_factory): """Fixture that provides instances of ``ResolverPlayground`` each one with one supported value for ``BINPKG_FORMAT``."""
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 92615cd37d7a1efce923c474e455f59fe61a589c
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 25 05:09:48 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Feb 25 08:24:59 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=92615cd3
_start_proc: Prevent premature ForkProcess garbage collection
In order to prevent the constructed ForkProcess instance from
being prematurely garbage collected, return a reference to the
caller inside of a _GCProtector object, preventing messages
reported in bug 925456 like this:
[ERROR] Task was destroyed but it is pending!
Fixes: a69c1b853a47 ("process.spawn: Use multiprocessing.Process for
returnproc")
Bug: https://bugs.gentoo.org/925456
Signed-off-by: Zac Medico gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1284
Signed-off-by: Sam James gentoo.org>
lib/portage/process.py | 41 +++--
1 file changed, 39 insertions(+), 2 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index d16262e75a..cc9ed7bf78 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -36,6 +36,7 @@ portage.proxy.lazyimport.lazyimport(
from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
from portage.exception import CommandNotFound
+from portage.proxy.objectproxy import ObjectProxy
from portage.util._ctypes import find_library, LoadLibrary, ctypes
try:
@@ -1493,8 +1494,44 @@ def _start_proc(
proc.start()
# ForkProcess conveniently holds a MultiprocessingProcess
-# instance that is suitable to return here.
-return proc._proc
+# instance that is suitable to return here, but use _GCProtector
+# to protect the ForkProcess instance from being garbage collected
+# and triggering messages like this (bug 925456):
+# [ERROR] Task was destroyed but it is pending!
+return _GCProtector(proc._proc, proc.async_wait)
+
+
+class _GCProtector(ObjectProxy):
+"""
+Proxy a target object, and also hold a reference to something
+extra in order to protect it from garbage collection. Override
+the wait method to first call target's wait method and then
+wait for extra (a coroutine function) before returning the result.
+"""
+
+__slots__ = ("_extra", "_target")
+
+def __init__(self, target, extra):
+super().__init__()
+object.__setattr__(self, "_target", target)
+object.__setattr__(self, "_extra", extra)
+
+def _get_target(self):
+return object.__getattribute__(self, "_target")
+
+def __getattribute__(self, attr):
+if attr == "wait":
+return object.__getattribute__(self, attr)
+return getattr(object.__getattribute__(self, "_target"), attr)
+
+async def wait(self):
+"""
+Wrap the target's wait method to also wait for an extra
+coroutine function.
+"""
+result = await object.__getattribute__(self, "_target").wait()
+await object.__getattribute__(self, "_extra")()
+return result
def find_binary(binary):
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/
commit: 9e84ef57ba747766c9147c1ac1b247faa1f05956
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 25 00:31:13 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 25 00:42:46 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9e84ef57
testSpawnReturnProcTerminate: Fix integer in spawn command argument
The invalid integer in the spawn command argument intermittently
triggered this error when SIGTERM did not arrive until after the
exec call:
- Captured stderr call -
Process ForkProcess-23:
Traceback (most recent call last):
File "/home/runner/work/portage/portage/lib/portage/process.py", line 818, in
_exec_wrapper
_exec(
File "/home/runner/work/portage/portage/lib/portage/process.py", line 1068,
in _exec
_exec2(
File "/home/runner/work/portage/portage/lib/portage/process.py", line 1166,
in _exec2
os.execve(binary, myargs, env)
TypeError: expected str, bytes or os.PathLike object, not int
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File
"/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/process.py",
line 314, in _bootstrap
self.run()
File
"/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/process.py",
line 108, in run
self._target(*self._args, **self._kwargs)
File
"/home/runner/work/portage/portage/lib/portage/util/_async/ForkProcess.py",
line 328, in _bootstrap
sys.exit(target(*(args or []), **(kwargs or {})))
^^^
File "/home/runner/work/portage/portage/lib/portage/process.py", line 1441,
in __call__
return self._target(*args, **kwargs)
^
File "/home/runner/work/portage/portage/lib/portage/process.py", line 853, in
_exec_wrapper
writemsg(f"{e}:\n {' '.join(mycommand)}\n", noiselevel=-1)
^^^
TypeError: sequence item 1: expected str instance, int found
Bug: https://bugs.gentoo.org/916566#c20
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/process/test_spawn_returnproc.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/tests/process/test_spawn_returnproc.py
b/lib/portage/tests/process/test_spawn_returnproc.py
index 6d823d9c3d..8fbf54d0d2 100644
--- a/lib/portage/tests/process/test_spawn_returnproc.py
+++ b/lib/portage/tests/process/test_spawn_returnproc.py
@@ -32,7 +32,7 @@ class SpawnReturnProcTestCase(TestCase):
loop = global_event_loop()
async def watch_pid():
-proc = spawn([sleep_binary, ], returnproc=True)
+proc = spawn([sleep_binary, ""], returnproc=True)
proc.terminate()
self.assertEqual(await proc.wait(), -signal.SIGTERM)
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 3f4250dc7d32e9915224b1c9c4bc04c2740abcda
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 23 20:35:04 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 24 20:07:39 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3f4250dc
process.spawn: Fix logic for missing libc.unshare on musl
Fix unshare_* variables to be False when the libc is missing,
libc.unshare is missing, or libc.unshare fails.
Also, if socket.sethostname is missing then _exec2 needs
libc for the network-sandbox sethostname call which is wrapped
by a blanket Exception handler.
Fixes: 419cce79f908 ("process._exec: Use _start_fork for os.fork() error
handling")
Bug: https://bugs.gentoo.org/925311
Signed-off-by: Zac Medico gentoo.org>
lib/portage/process.py | 213 +
1 file changed, 110 insertions(+), 103 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index f4758c824c..d16262e75a 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -956,114 +956,119 @@ def _exec(
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
# Unshare (while still uid==0)
+have_unshare = False
+libc = None
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
-# unshare() may not be supported by libc
-if not hasattr(libc, "unshare"):
-unshare_net = False
-unshare_ipc = False
-unshare_mount = False
-unshare_pid = False
-else:
-# Since a failed unshare call could corrupt process
-# state, first validate that the call can succeed.
-# The parent process should call _unshare_validate
-# before it forks, so that all child processes can
-# reuse _unshare_validate results that have been
-# cached by the parent process.
-errno_value = _unshare_validate(unshare_flags)
-if errno_value == 0 and libc.unshare(unshare_flags) != 0:
-errno_value = ctypes.get_errno()
-if errno_value != 0:
-involved_features = []
-if unshare_ipc:
-involved_features.append("ipc-sandbox")
-if unshare_mount:
-involved_features.append("mount-sandbox")
-if unshare_net:
-involved_features.append("network-sandbox")
-if unshare_pid:
-involved_features.append("pid-sandbox")
-
-writemsg(
-'Unable to unshare: %s (for FEATURES="%s")\n'
-% (
-errno.errorcode.get(errno_value, "?"),
-" ".join(involved_features),
-),
-noiselevel=-1,
-)
-else:
-if unshare_pid:
-# pid namespace requires us to become init
-binary, myargs = (
-portage._python_interpreter,
-[
-portage._python_interpreter,
-os.path.join(portage._bin_path,
"pid-ns-init"),
-_unicode_encode("" if uid is None else
str(uid)),
-_unicode_encode("" if gid is None else
str(gid)),
-_unicode_encode(
-""
-if groups is None
-else ",".join(str(group) for group in
groups)
-),
-_unicode_encode(
-"" if umask is None else str(umask)
-),
-_unicode_encode(
-",".join(str(fd) for fd in fd_pipes)
-),
-binary,
-]
-+ myargs,
-)
-uid = None
-gid = None
-groups = None
-umask = None
-
-# Use _start_fork for os.fork() error handling,
ensuring
-# that if exec fails then the child
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/emerge/
commit: 01d06eb1d9dc8c4b16cbc9a6567ed0c07df5901a
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 24 02:45:58 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 24 03:28:24 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=01d06eb1
Fix python 3.9 CI jobs since 92ff02b9189f
Use emerge -e to fix this error we've seen since
92ff02b9189f for python 3.9 CI jobs:
https://github.com/gentoo/portage/actions/runs/8014796128/job/21893963019
test_portage_baseline[binhost emerge-gpkg] - AssertionError: 'emerge' failed
with args '('-e', '--getbinpkgonly', 'dev-libs/A')'
emerge: there are no binary packages to satisfy "dev-libs/B[flag]".
(dependency required by "dev-libs/A-1::test_repo" [binary])
(dependency required by "dev-libs/A" [argument])
Fixes: 92ff02b9189f ("emerge: Skip installed packages with emptytree in
depgraph selection")
Bug: https://bugs.gentoo.org/651018
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/emerge/conftest.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/tests/emerge/conftest.py
b/lib/portage/tests/emerge/conftest.py
index d9aec7041e..580d1e09ab 100644
--- a/lib/portage/tests/emerge/conftest.py
+++ b/lib/portage/tests/emerge/conftest.py
@@ -805,7 +805,7 @@ def _generate_all_baseline_commands(playground, binhost):
test_commands["binhost emerge"] = Noop()
else:
# The next emerge has been added to split this test from the rest:
-make_package = Emerge("--buildpkg", "dev-libs/A")
+make_package = Emerge("-e", "--buildpkg", "dev-libs/A")
getbinpkgonly = Emerge(
"-e",
"--getbinpkgonly",
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/, lib/portage/util/_eventloop/
commit: 3cc986f87ddda86ee93770e03cca06346aee54c5
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 23 06:06:14 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 23 06:48:29 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3cc986f8
AsyncioEventLoop: Call process.run_exitfuncs() before close
For the event loop running in the main thread, call
process.run_exitfuncs() before close with the event loop
running so that anything attached can clean itself up (like
the socks5 ProxyManager for bug 925240). This is necessary
because process.spawn uses the event loop to implement the
new returnproc parameter related to bug 916566.
Bug: https://bugs.gentoo.org/916566
Bug: https://bugs.gentoo.org/925240
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_socks5.py | 51 ++-
lib/portage/util/_eventloop/asyncio_event_loop.py | 44 +++
lib/portage/util/socks5.py| 16 ++-
3 files changed, 101 insertions(+), 10 deletions(-)
diff --git a/lib/portage/tests/util/test_socks5.py
b/lib/portage/tests/util/test_socks5.py
index 7b33cb3f6b..4a6d08169d 100644
--- a/lib/portage/tests/util/test_socks5.py
+++ b/lib/portage/tests/util/test_socks5.py
@@ -12,6 +12,8 @@ import time
import portage
from portage.tests import TestCase
from portage.util import socks5
+from portage.util.futures.executor.fork import ForkExecutor
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.const import PORTAGE_BIN_PATH
from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -189,8 +191,10 @@ class Socks5ServerTestCase(TestCase):
path = "/index.html"
proxy = None
tempdir = tempfile.mkdtemp()
+previous_exithandlers = portage.process._exithandlers
try:
+portage.process._exithandlers = []
with AsyncHTTPServer(host, {path: content}, loop) as server:
settings = {
"PORTAGE_TMPDIR": tempdir,
@@ -211,5 +215,50 @@ class Socks5ServerTestCase(TestCase):
self.assertEqual(result, content)
finally:
-await socks5.proxy.stop()
+try:
+# Also run_exitfuncs to test atexit hook cleanup.
+await socks5.proxy.stop()
+self.assertNotEqual(portage.process._exithandlers, [])
+portage.process.run_exitfuncs()
+self.assertEqual(portage.process._exithandlers, [])
+finally:
+portage.process._exithandlers = previous_exithandlers
+shutil.rmtree(tempdir)
+
+
+class Socks5ServerLoopCloseTestCase(TestCase):
+"""
+For bug 925240, test that the socks5 proxy is automatically
+terminated when the main event loop is closed, using a subprocess
+for isolation.
+"""
+
+def testSocks5ServerLoopClose(self):
+asyncio.run(self._testSocks5ServerLoopClose())
+
+async def _testSocks5ServerLoopClose(self):
+loop = asyncio.get_running_loop()
+self.assertEqual(
+await loop.run_in_executor(
+ForkExecutor(loop=loop),
self._testSocks5ServerLoopCloseSubprocess
+),
+True,
+)
+
+@staticmethod
+def _testSocks5ServerLoopCloseSubprocess():
+loop = global_event_loop()
+tempdir = tempfile.mkdtemp()
+try:
+settings = {
+"PORTAGE_TMPDIR": tempdir,
+"PORTAGE_BIN_PATH": PORTAGE_BIN_PATH,
+}
+
+socks5.get_socks5_proxy(settings)
+loop.run_until_complete(socks5.proxy.ready())
+finally:
+loop.close()
shutil.rmtree(tempdir)
+
+return not socks5.proxy.is_running()
diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py
b/lib/portage/util/_eventloop/asyncio_event_loop.py
index b9e96bb20e..ee9e4c60ef 100644
--- a/lib/portage/util/_eventloop/asyncio_event_loop.py
+++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
@@ -1,8 +1,9 @@
-# Copyright 2018-2023 Gentoo Authors
+# Copyright 2018-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
import signal
+import threading
import asyncio as _real_asyncio
from asyncio.events import AbstractEventLoop as _AbstractEventLoop
@@ -14,6 +15,7 @@ except ImportError:
PidfdChildWatcher = None
import portage
+from portage.util import socks5
class AsyncioEventLoop(_AbstractEventLoop):
@@ -25,18 +27,14 @@ class AsyncioEventLoop(_AbstractEventLoop):
def __init__(self, loop=None):
loop = loop or _real_asyncio.get_event_loop()
self._loop = loop
-self.run_until_complete = (
-self._run_until_complete
-if portage._internal_caller
-else loop.run_until_complete
-)
+self.run_until_complete = self._run_until_complete
[gentoo-commits] proj/portage:master commit in: /, lib/portage/tests/resolver/, lib/_emerge/
commit: 92ff02b9189f8350f44e134d538319e4037f3f71
Author: Gábor Oszkár Dénes protonmail com>
AuthorDate: Wed Feb 14 17:40:24 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Feb 23 04:39:26 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=92ff02b9
emerge: Skip installed packages with emptytree in depgraph selection
Running emerge with emptytree tries to find the best match for every
atom it needs to install. Sometimes the best matches would be
already installed packages (with `operation=nomerge`), but these
packages would be silently skipped with full emptytree installation.
This change makes sure that emerge attempts to install every package.
If the package has unmet requirements, emerge will complain.
Bug: https://bugs.gentoo.org/651018
Signed-off-by: Gábor Oszkár Dénes protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1272
Signed-off-by: Sam James gentoo.org>
NEWS | 6 +
lib/_emerge/depgraph.py| 13 ++
lib/portage/tests/resolver/test_depth.py | 8 +-
.../test_emptytree_reinstall_unsatisfiability.py | 137 +
lib/portage/tests/resolver/test_useflags.py| 6 +-
5 files changed, 166 insertions(+), 4 deletions(-)
diff --git a/NEWS b/NEWS
index 3fbc727861..94be26de84 100644
--- a/NEWS
+++ b/NEWS
@@ -6,6 +6,12 @@ Release notes take the form of the following optional
categories:
* Bug fixes
* Cleanups
+portage-3.0.63 (UNRELEASED)
+--
+
+Bug fixes:
+* emerge: Skip installed packages with emptytree in depgraph selection (bug
#651018).
+
portage-3.0.62 (2024-02-22)
--
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 70b83ee1f4..ea96bd58c4 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -7639,6 +7639,19 @@ class depgraph:
if pkg.installed and root_slot in
self._rebuild.reinstall_list:
continue
+if (
+empty
+and pkg.installed
+and not
self._frozen_config.excluded_pkgs.findAtomForPackage(
+pkg, modified_use=self._pkg_use_enabled(pkg)
+)
+):
+# With --emptytree option we assume no packages
+# are installed, so we do not select them.
+# But we allow installed packages to satisfy
dependency requirements
+# if they're explicitly excluded, so we allow them to
be selected.
+continue
+
if (
not pkg.installed
and
self._frozen_config.excluded_pkgs.findAtomForPackage(
diff --git a/lib/portage/tests/resolver/test_depth.py
b/lib/portage/tests/resolver/test_depth.py
index 9c5289f7d0..ab5f8e7ec3 100644
--- a/lib/portage/tests/resolver/test_depth.py
+++ b/lib/portage/tests/resolver/test_depth.py
@@ -1,4 +1,4 @@
-# Copyright 2011-2020 Gentoo Authors
+# Copyright 2011-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -318,6 +318,12 @@ class ResolverDepthTestCase(TestCase):
"sys-fs/udev-164",
],
),
+ResolverPlaygroundTestCase(
+["@world"],
+options={"--emptytree": True, "--exclude": ["dev-libs/B"]},
+success=True,
+mergelist=["dev-libs/C-2", "dev-libs/A-2"],
+),
)
playground = ResolverPlayground(
diff --git
a/lib/portage/tests/resolver/test_emptytree_reinstall_unsatisfiability.py
b/lib/portage/tests/resolver/test_emptytree_reinstall_unsatisfiability.py
new file mode 100644
index 00..fcdc01d7f1
--- /dev/null
+++ b/lib/portage/tests/resolver/test_emptytree_reinstall_unsatisfiability.py
@@ -0,0 +1,137 @@
+# Copyright 2024 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+ResolverPlayground,
+ResolverPlaygroundTestCase,
+)
+
+
+class EmptytreeReinstallUnsatisfiabilityTestCase(TestCase):
+def testEmptytreeReinstallUnsatisfiability(self):
+"""
+Tests to check if emerge fails and complains when --emptytree
+package dependency graph reinstall is unsatisfied, even if the already
+installed packages successfully satisfy the dependency tree.
+
+See bug #651018 where emerge silently skips package
+reinstalls because of unsatisfied use flag requirements.
+"""
+ebuilds = {
+"dev-libs/A-1": {
+"DEPEND": "dev-libs/B",
+"RDEPEND": "dev-libs/B",
+"EAPI": "2",
+},
+
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: d718cea94a180042b2285698b2c19113c5d25987 Author: Zac Medico gentoo org> AuthorDate: Thu Feb 22 06:41:49 2024 + Commit: Zac Medico gentoo org> CommitDate: Thu Feb 22 07:28:38 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d718cea9 _get_running_loop: Support real asyncio.run When called via the real asyncio.run implementation, wrap the running asyncio loop. Otherwise, it's not possible to call portage libraries via the real asyncio.run without triggering Future "attached to a different loop" errors. Bug: https://bugs.gentoo.org/761538 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/_asyncio/__init__.py | 28 +-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index 22241f335d..4eecc46a89 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -325,13 +325,37 @@ def _safe_loop(): def _get_running_loop(): +""" +This calls the real asyncio get_running_loop() and wraps that with +portage's internal AsyncioEventLoop wrapper. If there is no running +asyncio event loop but portage has a reference to another running +loop in this thread, then use that instead. + +This behavior enables portage internals to use the real asyncio.run +while remaining compatible with internal code that does not use the +real asyncio.run. +""" +try: +_loop = _real_asyncio.get_running_loop() +except RuntimeError: +_loop = None + with _thread_weakrefs.lock: if _thread_weakrefs.pid == portage.getpid(): try: loop = _thread_weakrefs.loops[threading.get_ident()] except KeyError: -return None -return loop if loop.is_running() else None +pass +else: +if _loop is loop._loop: +return loop +elif _loop is None: +return loop if loop.is_running() else None + +# If _loop it not None here it means it was probably a temporary +# loop created by asyncio.run, so we don't try to cache it, and +# just return a temporary wrapper. +return None if _loop is None else _AsyncioEventLoop(loop=_loop) def _thread_weakrefs_atexit():
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
commit: fbaaa4a733aaadc2744b656527756ac4e2b7ab58
Author: Zac Medico gentoo org>
AuthorDate: Thu Feb 22 06:47:33 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Feb 22 07:28:38 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fbaaa4a7
socks5: Use real asyncio.run
Use real asyncio.run to demonstrate that it is compatible with
portage internals. Since the socks5 ProxyManager uses the
process.spawn function, the internal _running_loop function
needs to return the correct loop for use in the wait method of
MultiprocessingProcess, or else it will lead to Future
"attached to a different loop" errors.
Bug: https://bugs.gentoo.org/761538
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_socks5.py | 45 +++
lib/portage/util/socks5.py| 30 ---
2 files changed, 46 insertions(+), 29 deletions(-)
diff --git a/lib/portage/tests/util/test_socks5.py
b/lib/portage/tests/util/test_socks5.py
index 987b41af29..7b33cb3f6b 100644
--- a/lib/portage/tests/util/test_socks5.py
+++ b/lib/portage/tests/util/test_socks5.py
@@ -1,6 +1,7 @@
-# Copyright 2019-2021 Gentoo Authors
+# Copyright 2019-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import asyncio
import functools
import shutil
import socket
@@ -10,7 +11,6 @@ import time
import portage
from portage.tests import TestCase
-from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util import socks5
from portage.const import PORTAGE_BIN_PATH
@@ -88,18 +88,20 @@ class AsyncHTTPServerTestCase(TestCase):
if f is not None:
f.close()
-def test_http_server(self):
+async def _test_http_server(self):
+asyncio.run(self._test_http_server())
+
+async def _test_http_server(self):
host = "127.0.0.1"
content = b"Hello World!\n"
path = "/index.html"
-loop = global_event_loop()
+
+loop = asyncio.get_running_loop()
for i in range(2):
with AsyncHTTPServer(host, {path: content}, loop) as server:
for j in range(2):
-result = loop.run_until_complete(
-loop.run_in_executor(
-None, self._fetch_directly, host,
server.server_port, path
-)
+result = await loop.run_in_executor(
+None, self._fetch_directly, host, server.server_port,
path
)
self.assertEqual(result, content)
@@ -177,7 +179,10 @@ class Socks5ServerTestCase(TestCase):
return f.read()
def test_socks5_proxy(self):
-loop = global_event_loop()
+asyncio.run(self._test_socks5_proxy())
+
+async def _test_socks5_proxy(self):
+loop = asyncio.get_running_loop()
host = "127.0.0.1"
content = b"Hello World!"
@@ -193,20 +198,18 @@ class Socks5ServerTestCase(TestCase):
}
proxy = socks5.get_socks5_proxy(settings)
-loop.run_until_complete(socks5.proxy.ready())
-
-result = loop.run_until_complete(
-loop.run_in_executor(
-None,
-self._fetch_via_proxy,
-proxy,
-host,
-server.server_port,
-path,
-)
+await socks5.proxy.ready()
+
+result = await loop.run_in_executor(
+None,
+self._fetch_via_proxy,
+proxy,
+host,
+server.server_port,
+path,
)
self.assertEqual(result, content)
finally:
-socks5.proxy.stop()
+await socks5.proxy.stop()
shutil.rmtree(tempdir)
diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py
index 6c68ff4106..74592aeefe 100644
--- a/lib/portage/util/socks5.py
+++ b/lib/portage/util/socks5.py
@@ -2,15 +2,16 @@
# Copyright 2015-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import asyncio
import errno
import os
import socket
+from typing import Union
import portage.data
from portage import _python_interpreter
from portage.data import portage_gid, portage_uid, userpriv_groups
from portage.process import atexit_register, spawn
-from portage.util.futures import asyncio
class ProxyManager:
@@ -57,23 +58,36 @@ class ProxyManager:
**spawn_kwargs,
)
-def stop(self):
+def stop(self) -> Union[None, asyncio.Future]:
"""
Stop the SOCKSv5 server.
+
+If there is a running asyncio event loop then asyncio.Future is
+returned which should be used to wait
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/package/ebuild/, ...
commit: 18cdb6331a66c1cc92f296b1aaf0538f63586875
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 08:44:50 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=18cdb633
EbuildPhase: async_check_locale
Change config.environ() check_locale calls to async_check_locale
calls in the EbuildPhase _async_start method in order to eliminate
synchronous waiting for child processes in the main event loop
thread.
Bug: https://bugs.gentoo.org/923841
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py| 4
lib/_emerge/EbuildPhase.py| 28 ++-
lib/portage/package/ebuild/config.py | 26 +++--
lib/portage/util/futures/_asyncio/__init__.py | 9 +
lib/portage/util/locale.py| 28 ++-
5 files changed, 70 insertions(+), 25 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index fc64d84c94..54177840c7 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,6 +8,7 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"_emerge.EbuildPhase:_setup_locale",
"portage.package.ebuild._metadata_invalid:eapi_invalid",
)
from portage import os
@@ -83,6 +84,9 @@ class EbuildMetadataPhase(SubProcess):
settings.setcpv(self.cpv)
settings.configdict["pkg"]["EAPI"] = parsed_eapi
+# This requires above setcpv and EAPI setup.
+await _setup_locale(self.settings)
+
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 3b366f39c7..b472803438 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import (
_prepare_fake_distdir,
_prepare_fake_filesdir,
)
+from portage.eapi import _get_eapi_attrs
from portage.util import writemsg, ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport(
+ "_post_src_install_write_metadata,"
+ "_preinst_bsdflags",
"portage.util.futures.unix_events:_set_nonblocking",
+"portage.util.locale:async_check_locale,split_LC_ALL",
)
from portage import os
from portage import _encodings
from portage import _unicode_encode
+async def _setup_locale(settings):
+eapi_attrs = _get_eapi_attrs(settings["EAPI"])
+if eapi_attrs.posixish_locale:
+split_LC_ALL(settings)
+settings["LC_COLLATE"] = "C"
+# check_locale() returns None when check can not be executed.
+if await async_check_locale(silent=True, env=settings.environ()) is
False:
+# try another locale
+for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
+settings["LC_CTYPE"] = l
+if await async_check_locale(silent=True,
env=settings.environ()):
+# TODO: output the following only once
+# writemsg(
+# _("!!! LC_CTYPE unsupported, using %s instead\n")
+# % self.settings["LC_CTYPE"]
+# )
+break
+else:
+raise AssertionError("C locale did not pass the test!")
+
+
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") +
("_ebuild_lock",)
@@ -95,6 +118,9 @@ class EbuildPhase(CompositeTask):
self._start_task(AsyncTaskFuture(future=future),
self._async_start_exit)
async def _async_start(self):
+
+await _setup_locale(self.settings)
+
need_builddir = self.phase not in
EbuildProcess._phases_without_builddir
if need_builddir:
diff --git a/lib/portage/package/ebuild/config.py
b/lib/portage/package/ebuild/config.py
index c89354cbf7..bafdc55a08 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -29,7 +29,6 @@ portage.proxy.lazyimport.lazyimport(
"portage.dbapi.vartree:vartree",
"portage.package.ebuild.doebuild:_phase_func_map",
"portage.util.compression_probe:_compressors",
-"portage.util.locale:check_locale,split_LC_ALL",
)
from portage import bsd_chflags, load_mod, os, selinux, _unicode_decode
from portage.const import (
@@ -3371,20 +3370,17 @@ class config:
mydict["EBUILD_PHASE_FUNC"] = phase_func
if eapi_attrs.posixish_locale:
-split_LC_ALL(mydict)
-
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: a42c2164ada634262ae1f791ad60298fe3468a94
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 03:39:35 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a42c2164
asyncio: Wrap asyncio.Lock for python 3.9 compat
Wrap asyncio.Lock for compatibility with python 3.9 where the
deprecated loop parameter is required in order to avoid "got
Future attached to a different loop" errors.
The pordbapi async_aux_get method can use asyncio.Lock to
serialize access to its doebuild_settings attribute in order
to prevent issues like bug 924319.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/_asyncio/__init__.py | 17 +
1 file changed, 17 insertions(+)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index 8f1b8e8275..b6481c281e 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -9,6 +9,7 @@ __all__ = (
"CancelledError",
"Future",
"InvalidStateError",
+"Lock",
"TimeoutError",
"get_child_watcher",
"get_event_loop",
@@ -22,6 +23,7 @@ __all__ = (
"wait_for",
)
+import sys
import types
import weakref
@@ -35,6 +37,7 @@ from asyncio import (
FIRST_EXCEPTION,
Future,
InvalidStateError,
+Lock as _Lock,
shield,
TimeoutError,
wait_for,
@@ -159,6 +162,20 @@ def iscoroutinefunction(func):
return False
+class Lock(_Lock):
+"""
+Inject loop parameter for python3.9 or less in order to avoid
+"got Future attached to a different loop" errors.
+"""
+
+def __init__(self, **kwargs):
+if sys.version_info >= (3, 10):
+kwargs.pop("loop", None)
+elif "loop" not in kwargs:
+kwargs["loop"] = _safe_loop()._loop
+super().__init__(**kwargs)
+
+
class Task(Future):
"""
Schedule the execution of a coroutine: wrap it in a future. A task
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/, lib/portage/tests/update/, lib/portage/dbapi/
commit: 389bb304abf5705b9f0e9e75982a682f193af985
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 04:35:02 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=389bb304
async_aux_get: Use EbuildMetadataPhase deallocate_config future
For the portdbapi async_aux_get method, there is not a very
good place to store a config pool, so instead use asyncio.Lock
to manage access to the portdbapi doebuild_settings attribute
when using the main event loop in the main thread. For other
threads, clone a config instance since we do not have a
thread-safe config pool. This cloning is expensive, but since
portage internals do not trigger this case, it suffices for now
(an AssertionError ensures that internals do not trigger it).
For the main event loop running in the main thread, performance
with the asyncio.Lock should not be significantly different to
performance prior to commit c95fc64abf96, since check_locale
results are typically cached and before there was only a single
shared doebuild_settings instance with access serialized via
the EbuildMetadataPhase _start method.
Update async_aux_get callers to use asyncio.ensure_future on
the returned coroutine when needed, since it used to return
a future instead of a coroutine, and sometimes a future is
needed for add_done_callback usage.
In the portdbapi async_fetch_map method, fix a broken reference
to "future" which should have been "aux_get_future", an error
discovered while testing this patch.
Bug: https://bugs.gentoo.org/924319
Fixes: c95fc64abf96 ("EbuildPhase: async_check_locale")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_emirrordist/FetchIterator.py | 10 ++-
lib/portage/dbapi/porttree.py | 129 --
lib/portage/tests/update/test_move_ent.py | 3 +
3 files changed, 97 insertions(+), 45 deletions(-)
diff --git a/lib/portage/_emirrordist/FetchIterator.py
b/lib/portage/_emirrordist/FetchIterator.py
index eaf3e53596..e4fdd092af 100644
--- a/lib/portage/_emirrordist/FetchIterator.py
+++ b/lib/portage/_emirrordist/FetchIterator.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2018 Gentoo Foundation
+# Copyright 2013-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import threading
@@ -14,6 +14,7 @@ from portage.exception import PortageException,
PortageKeyError
from portage.package.ebuild.fetch import DistfileName
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.TaskScheduler import TaskScheduler
+from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from .FetchTask import FetchTask
from _emerge.CompositeTask import CompositeTask
@@ -276,8 +277,11 @@ def _async_fetch_tasks(config, hash_filter, repo_config,
digests_future, cpv, lo
result.set_result(fetch_tasks)
def future_generator():
-yield config.portdb.async_aux_get(
-cpv, ("RESTRICT",), myrepo=repo_config.name, loop=loop
+yield asyncio.ensure_future(
+config.portdb.async_aux_get(
+cpv, ("RESTRICT",), myrepo=repo_config.name, loop=loop
+),
+loop,
)
yield config.portdb.async_fetch_map(cpv, mytree=repo_config.location,
loop=loop)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 61d431f917..4eebe1183f 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2021 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["close_portdbapi_caches", "FetchlistDict", "portagetree",
"portdbapi"]
@@ -41,7 +41,9 @@ from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
+import contextlib
import os as _os
+import threading
import traceback
import warnings
import errno
@@ -239,6 +241,7 @@ class portdbapi(dbapi):
# this purpose because doebuild makes many changes to the config
# instance that is passed in.
self.doebuild_settings = config(clone=self.settings)
+self._doebuild_settings_lock = asyncio.Lock()
self.depcachedir = os.path.realpath(self.settings.depcachedir)
if os.environ.get("SANDBOX_ON") == "1":
@@ -356,6 +359,17 @@ class portdbapi(dbapi):
self._better_cache = None
self._broken_ebuilds = set()
+def __getstate__(self):
+state = self.__dict__.copy()
+# These attributes are not picklable, so they are automatically
+# regenerated after unpickling.
+state["_doebuild_settings_lock"] = None
+return state
+
+def __setstate__(self, state):
+self.__dict__.update(state)
+self._doebuild_settings_lock =
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dbapi/, lib/portage/tests/resolver/
commit: 0dedea99ac13e0e75a83a78890ed73bced1b950b
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 04:21:26 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0dedea99
ResolverPlayground: Use egencache to create manifests
Make the ResolverPlayground _create_ebuild_manifests method
call egencache --jobs, which reliably triggers the KeyError
from bug 924319 for multiple tests:
lib/portage/tests/bin/test_doins.py::DoIns::testDoInsFallback Exception in
callback EbuildMetadataPhase._async_start_done()
handle: )>
Traceback (most recent call last):
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "lib/_emerge/EbuildMetadataPhase.py", line 154, in _async_start_done
future.cancelled() or future.result()
^^^
File "lib/_emerge/EbuildMetadataPhase.py", line 130, in _async_start
retval = portage.doebuild(
^
File "lib/portage/package/ebuild/doebuild.py", line 1030, in doebuild
doebuild_environment(
File "lib/portage/package/ebuild/doebuild.py", line 519, in
doebuild_environment
eapi = mysettings.configdict["pkg"]["EAPI"]
File "lib/portage/util/__init__.py", line 1684, in __getitem__
return UserDict.__getitem__(self, item_key)
File "lib/portage/cache/mappings.py", line 175, in __getitem__
return self.data[key]
~^
KeyError: 'EAPI'
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/dbapi/test_portdb_cache.py | 4 ++-
lib/portage/tests/resolver/ResolverPlayground.py | 38 +---
2 files changed, 23 insertions(+), 19 deletions(-)
diff --git a/lib/portage/tests/dbapi/test_portdb_cache.py
b/lib/portage/tests/dbapi/test_portdb_cache.py
index c7c6913b49..c24a4f2098 100644
--- a/lib/portage/tests/dbapi/test_portdb_cache.py
+++ b/lib/portage/tests/dbapi/test_portdb_cache.py
@@ -1,6 +1,7 @@
-# Copyright 2012-2023 Gentoo Authors
+# Copyright 2012-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import shutil
import subprocess
import sys
import textwrap
@@ -63,6 +64,7 @@ class PortdbCacheTestCase(TestCase):
python_cmd = (portage_python, "-b", "-Wd", "-c")
test_commands = (
+(lambda: shutil.rmtree(md5_cache_dir) or True,),
(lambda: not os.path.exists(pms_cache_dir),),
(lambda: not os.path.exists(md5_cache_dir),),
python_cmd
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index 2d26012873..75c86b615c 100644
--- a/lib/portage/tests/resolver/ResolverPlayground.py
+++ b/lib/portage/tests/resolver/ResolverPlayground.py
@@ -3,6 +3,7 @@
import bz2
import fnmatch
+import subprocess
import tempfile
import portage
@@ -18,8 +19,6 @@ from portage.const import (
from portage.process import find_binary
from portage.dep import Atom, _repo_separator
from portage.dbapi.bintree import binarytree
-from portage.package.ebuild.config import config
-from portage.package.ebuild.digestgen import digestgen
from portage._sets import load_default_config
from portage._sets.base import InternalPackageSet
from portage.tests import cnf_path
@@ -323,22 +322,25 @@ class ResolverPlayground:
f.write(misc_content)
def _create_ebuild_manifests(self, ebuilds):
-tmpsettings = config(clone=self.settings)
-tmpsettings["PORTAGE_QUIET"] = "1"
-for cpv in ebuilds:
-a = Atom("=" + cpv, allow_repo=True)
-repo = a.repo
-if repo is None:
-repo = "test_repo"
-
-repo_dir = self._get_repo_dir(repo)
-ebuild_dir = os.path.join(repo_dir, a.cp)
-ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] +
".ebuild")
-
-portdb = self.trees[self.eroot]["porttree"].dbapi
-tmpsettings["O"] = ebuild_dir
-if not digestgen(mysettings=tmpsettings, myportdb=portdb):
-raise AssertionError(f"digest creation failed for
{ebuild_path}")
+for repo_name in self._repositories:
+if repo_name == "DEFAULT":
+continue
+egencache_cmd = [
+"egencache",
+f"--repo={repo_name}",
+"--update",
+"--update-manifests",
+"--sign-manifests=n",
+"--strict-manifests=n",
+
f"--repositories-configuration={self.settings['PORTAGE_REPOSITORIES']}",
+f"--jobs={portage.util.cpuinfo.get_cpu_count()}",
+]
+result = subprocess.run(
+egencache_cmd,
+
[gentoo-commits] proj/portage:master commit in: lib/portage/cache/, lib/portage/tests/dbapi/
commit: 414234a218bc79564ab17312d5cc247a4c8091d7
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 03:30:00 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:30 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=414234a2
anydbm: Pickle support for multiprocessing spawn
The egencache usage in ResolverPlayground that was used to trigger
bug 924319 triggered a pickling error for AuxdbTestCase.test_anydbm
with the multiprocessing spawn start method, so fix the anydbm
cache module to omit the unpicklable database object from pickled
state, and regenerate it after unpickling.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico gentoo.org>
lib/portage/cache/anydbm.py | 17 -
lib/portage/tests/dbapi/test_auxdb.py | 4 +---
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/lib/portage/cache/anydbm.py b/lib/portage/cache/anydbm.py
index 94a270a483..ad7042ae41 100644
--- a/lib/portage/cache/anydbm.py
+++ b/lib/portage/cache/anydbm.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2020 Gentoo Authors
+# Copyright 2005-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# Author(s): Brian Harring (ferri...@gentoo.org)
@@ -67,6 +67,21 @@ class database(fs_template.FsBased):
raise cache_errors.InitializationError(self.__class__, e)
self._ensure_access(self._db_path)
+def __getstate__(self):
+state = self.__dict__.copy()
+# These attributes are not picklable, so they are automatically
+# regenerated after unpickling.
+state["_database__db"] = None
+return state
+
+def __setstate__(self, state):
+self.__dict__.update(state)
+mode = "w"
+if dbm.whichdb(self._db_path) in ("dbm.gnu", "gdbm"):
+# Allow multiple concurrent writers (see bug #53607).
+mode += "u"
+self.__db = dbm.open(self._db_path, mode, self._perms)
+
def iteritems(self):
# dbm doesn't implement items()
for k in self.__db.keys():
diff --git a/lib/portage/tests/dbapi/test_auxdb.py
b/lib/portage/tests/dbapi/test_auxdb.py
index 0de0123a5f..aac6ce361c 100644
--- a/lib/portage/tests/dbapi/test_auxdb.py
+++ b/lib/portage/tests/dbapi/test_auxdb.py
@@ -16,9 +16,7 @@ class AuxdbTestCase(TestCase):
from portage.cache.anydbm import database
except ImportError:
self.skipTest("dbm import failed")
-self._test_mod(
-"portage.cache.anydbm.database", multiproc=False, picklable=False
-)
+self._test_mod("portage.cache.anydbm.database", multiproc=False,
picklable=True)
def test_flat_hash_md5(self):
self._test_mod("portage.cache.flat_hash.md5_database")
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, bin/, man/
commit: 997058a825a340813532bef77a34425cf4a88eb2
Author: Michał Górny gentoo org>
AuthorDate: Thu Feb 15 15:33:03 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 21 02:13:45 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=997058a8
Support PROPERTIES=test_userpriv not to drop perms for tests
Support PROPERTIES=test_userpriv and a corresponding ALLOW_TEST=userpriv
to disable FEATURES=userpriv when running the test phase. This can be
used e.g. in dev-python/reflink that needs to be able to mount
filesystem on a loopback device for testing.
Bug: https://bugs.gentoo.org/924585
Signed-off-by: Michał Górny gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1274
Signed-off-by: Sam James gentoo.org>
bin/phase-functions.sh | 3 ++-
lib/portage/package/ebuild/config.py | 3 +++
lib/portage/package/ebuild/doebuild.py | 3 +++
man/ebuild.5 | 5 +
man/make.conf.5| 4
5 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index cd672a878c..ebcf5f242a 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -503,7 +503,8 @@ __dyn_test() {
fi
if has test ${PORTAGE_RESTRICT} && ! has all ${ALLOW_TEST} &&
- ! { has test_network ${PORTAGE_PROPERTIES} && has
network ${ALLOW_TEST}; }
+ ! { has test_network ${PORTAGE_PROPERTIES} && has
network ${ALLOW_TEST}; } &&
+ ! { has test_privileged ${PORTAGE_PROPERTIES} && has
privileged ${ALLOW_TEST}; }
then
einfo "Skipping make test/check due to ebuild restriction."
__vecho ">>> Test phase [disabled because of RESTRICT=test]:
${CATEGORY}/${PF}"
diff --git a/lib/portage/package/ebuild/config.py
b/lib/portage/package/ebuild/config.py
index d7b0ca5676..c89354cbf7 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -2114,6 +2114,9 @@ class config:
"test" in restrict
and not "all" in allow_test
and not ("test_network" in properties and "network" in
allow_test)
+and not (
+"test_privileged" in properties and "privileged" in
allow_test
+)
)
if restrict_test and "test" in self.features:
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 4cf155e033..bc51fdff2d 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -239,6 +239,9 @@ def _doebuild_spawn(phase, settings, actionmap=None,
**kwargs):
ebuild_sh_arg,
)
+if phase == "test" and "test_privileged" in
settings["PORTAGE_PROPERTIES"].split():
+kwargs["droppriv"] = False
+
settings["EBUILD_PHASE"] = phase
try:
return spawn(cmd, settings, **kwargs)
diff --git a/man/ebuild.5 b/man/ebuild.5
index f849f20a29..a32ba4828c 100644
--- a/man/ebuild.5
+++ b/man/ebuild.5
@@ -811,6 +811,11 @@ is installed.
The package manager may run tests that require an internet connection, even if
the ebuild has
.IR RESTRICT=test .
+.TP
+.I test_privileged
+The package manager may run tests that require superuser permissions, even if
+the ebuild has
+.IR RESTRICT=test .
.RE
.PD 1
.TP
diff --git a/man/make.conf.5 b/man/make.conf.5
index 23d8408544..e13f6eec4f 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -1323,6 +1323,10 @@ Run tests in packages specifying
\fBPROPERTIES\fR="\fBtest_network\fR". Note
that this will most likely cause Internet access during the test suite which
could cause additional costs, privacy concerns and intermittent test failures.
.TP
+.B privileged
+Run tests in packages specifying \fBPROPERTIES\fR="\fBtest_privileged\fR".
Note
+that this will cause the test suite to be run with superuser permissions.
+.TP
.RE
.TP
.B RESUMECOMMAND
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 58b4c156543705f631444040c2b962c4ac760f6a
Author: Sam James gentoo org>
AuthorDate: Wed Feb 21 02:07:59 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 21 02:08:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=58b4c156
gpkg: add missing newline to gpkg format error
Signed-off-by: Sam James gentoo.org>
lib/portage/gpkg.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index f1d8f97f8e..edb0e43fbf 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -2111,7 +2111,7 @@ class gpkg:
if self.basename and self.prefix and not
self.prefix.startswith(self.basename):
writemsg(
-colorize("WARN", f"Package basename mismatched, using
{self.prefix}")
+colorize("WARN", f"Package basename mismatched, using
{self.prefix}\n")
)
all_files = tar.getmembers()
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 14fe81d73c3330ec31bbc712ada40cc5bdda2c61
Author: Sam James gentoo org>
AuthorDate: Wed Feb 21 02:07:19 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 21 02:07:19 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=14fe81d7
binpkg: add missing newline to gpkg format error
Signed-off-by: Sam James gentoo.org>
lib/portage/binpkg.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
index 2078e3ca58..9ecd52cf3c 100644
--- a/lib/portage/binpkg.py
+++ b/lib/portage/binpkg.py
@@ -67,7 +67,7 @@ def get_binpkg_format(binpkg_path, check_file=False,
remote=False):
writemsg(
colorize(
"WARN",
-"File {} binpkg format mismatch, actual format: {}".format(
+"File {} binpkg format mismatch, actual format: {}\n".format(
binpkg_path, file_format
),
)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 038ad1029ea574b106906380e47479db1041bee2
Author: Zac Medico gentoo org>
AuthorDate: Wed Feb 14 05:55:31 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 14 06:04:22 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=038ad102
BuildLogger: Fix portage.locks._open_fds memory leak
The _file_close_wrapper __getattribute__ method needs to
be overridden to expose its close method, otherwise the
underlying file's close method is called and the closed
file object remains as a memory leak in the global
portage.locks._open_fds dict. For reference, see similar
classes like portage.util.atomic_ofstream which overrides
methods in the same way.
Bug: https://bugs.gentoo.org/919072
Fixes: df212738bbb2 ("BuildLogger: Close self._stdin after fork")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_async/BuildLogger.py | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/portage/util/_async/BuildLogger.py
b/lib/portage/util/_async/BuildLogger.py
index 9f8a21ab2b..0cfc90a942 100644
--- a/lib/portage/util/_async/BuildLogger.py
+++ b/lib/portage/util/_async/BuildLogger.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Gentoo Authors
+# Copyright 2020-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -31,6 +31,11 @@ class _file_close_wrapper(ObjectProxy):
def _get_target(self):
return object.__getattribute__(self, "_file")
+def __getattribute__(self, attr):
+if attr == "close":
+return object.__getattribute__(self, attr)
+return getattr(object.__getattribute__(self, "_file"), attr)
+
def close(self):
file = object.__getattribute__(self, "_file")
if not file.closed:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/tests/sets/files/, ...
commit: 3110ec376cbcb1f5b7fb82ba30ec958798bb32cf
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 00:46:52 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 13 05:04:40 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3110ec37
actions: Fix interaction between start-method and pytest-xdist
Use portage.test.TestCase setUp method to setup the multiprocessing
start method if needed. It needs to be done relatively late in order
to work with the pytest-xdist plugin due to execnet usage.
Bug: https://bugs.gentoo.org/914876
Signed-off-by: Zac Medico gentoo.org>
.github/workflows/ci.yml| 2 +-
lib/portage/tests/__init__.py | 12 +++-
lib/portage/tests/env/config/test_PortageModulesFile.py | 3 ++-
lib/portage/tests/news/test_NewsItem.py | 3 ++-
lib/portage/tests/sets/files/test_config_file_set.py| 3 ++-
lib/portage/tests/sets/files/test_static_file_set.py| 3 ++-
lib/portage/tests/sets/shell/test_shell.py | 4 ++--
lib/portage/tests/util/futures/test_retry.py| 1 +
8 files changed, 23 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 15da507238..5bffd97206 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -82,7 +82,6 @@ jobs:
mv "${bin_file}"{.new,}
fi
done < <(find bin -maxdepth 1 -type f)
- sed -i meson.build -e "s|'-m', 'pytest'|'-c', 'import
multiprocessing, sys, pytest; multiprocessing.set_start_method(\"spawn\",
force=True); sys.exit(pytest.console_main())'|"
- name: Test meson install --destdir /tmp/install-root
run: |
echo -e "[binaries]\npython = '$(command -v python)'" >
/tmp/native.ini
@@ -90,5 +89,6 @@ jobs:
meson install -C /tmp/build --destdir /tmp/install-root
- name: Run tests for ${{ matrix.python-version }}
run: |
+ [[ "${{ matrix.start-method }}" == "spawn" ]] && export
PORTAGE_MULTIPROCESSING_START_METHOD=spawn
export PYTEST_ADDOPTS="-vv -ra -l -o console_output_style=count -n
$(nproc) --dist=worksteal"
meson test -C /tmp/build --verbose
diff --git a/lib/portage/tests/__init__.py b/lib/portage/tests/__init__.py
index ef59852989..23dd366d89 100644
--- a/lib/portage/tests/__init__.py
+++ b/lib/portage/tests/__init__.py
@@ -1,8 +1,9 @@
# tests/__init__.py -- Portage Unit Test functionality
-# Copyright 2006-2023 Gentoo Authors
+# Copyright 2006-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import argparse
+import multiprocessing
import sys
import time
import unittest
@@ -79,6 +80,15 @@ class TestCase(unittest.TestCase):
self.bindir = cnf_bindir
self.sbindir = cnf_sbindir
+def setUp(self):
+"""
+Setup multiprocessing start method if needed. It needs to be
+done relatively late in order to work with the pytest-xdist
+plugin due to execnet usage.
+"""
+if os.environ.get("PORTAGE_MULTIPROCESSING_START_METHOD") == "spawn":
+multiprocessing.set_start_method("spawn", force=True)
+
def assertRaisesMsg(self, msg, excClass, callableObj, *args, **kwargs):
"""Fail unless an exception of class excClass is thrown
by callableObj when invoked with arguments args and keyword
diff --git a/lib/portage/tests/env/config/test_PortageModulesFile.py
b/lib/portage/tests/env/config/test_PortageModulesFile.py
index f9879df687..bca86e0e6c 100644
--- a/lib/portage/tests/env/config/test_PortageModulesFile.py
+++ b/lib/portage/tests/env/config/test_PortageModulesFile.py
@@ -1,4 +1,4 @@
-# Copyright 2006-2009 Gentoo Foundation
+# Copyright 2006-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage import os
@@ -13,6 +13,7 @@ class PortageModulesFileTestCase(TestCase):
modules = ["spanky", "zmedico", "antarus", "ricer", "5", "6"]
def setUp(self):
+super().setUp()
self.items = {}
for k, v in zip(self.keys + self.invalid_keys, self.modules):
self.items[k] = v
diff --git a/lib/portage/tests/news/test_NewsItem.py
b/lib/portage/tests/news/test_NewsItem.py
index a7903f07e4..7a8393c51f 100644
--- a/lib/portage/tests/news/test_NewsItem.py
+++ b/lib/portage/tests/news/test_NewsItem.py
@@ -1,5 +1,5 @@
# test_NewsItem.py -- Portage Unit Testing Functionality
-# Copyright 2007-2023 Gentoo Authors
+# Copyright 2007-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -114,6 +114,7 @@ class NewsItemTestCase(TestCase):
}
def setUp(self) -> None:
+super().setUp()
self.profile_base = "/var/db/repos/gentoo/profiles/default-linux"
self.profile = f"{self.profile_base}/x86/2007.0/"
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, lib/_emerge/, lib/portage/util/futures/_asyncio/, ...
commit: 5c528b1cf44f30d80a3ca5620a810e4fe2bd66f1
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 04:47:53 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 13 05:02:14 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5c528b1c
Revert "EbuildPhase: async_check_locale"
This reverts commit c95fc64abf9698263090b3ffd4a056e989dd2be1
since we had assumed EbuildMetadataPhase._start would serialize
access to the portdbapi doebuild_settings attribute, and that
assumption broke when _async_start was introduced in order to
call async_check_locale.
Bug: https://bugs.gentoo.org/923841
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py| 21
lib/_emerge/EbuildPhase.py| 28 +--
lib/portage/package/ebuild/config.py | 26 ++---
lib/portage/util/futures/_asyncio/__init__.py | 9 -
lib/portage/util/locale.py| 28 +--
5 files changed, 25 insertions(+), 87 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index 53b7ad9624..f4f685e81c 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,14 +8,12 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
-"_emerge.EbuildPhase:_setup_locale",
"portage.package.ebuild._metadata_invalid:eapi_invalid",
)
from portage import os
from portage import _encodings
from portage import _unicode_decode
from portage import _unicode_encode
-from portage.util.futures import asyncio
import fcntl
@@ -46,12 +44,6 @@ class EbuildMetadataPhase(SubProcess):
_files_dict = slot_dict_class(_file_names, prefix="")
def _start(self):
-asyncio.ensure_future(
-self._async_start(), loop=self.scheduler
-).add_done_callback(self._async_start_done)
-
-async def _async_start(self):
-
ebuild_path = self.ebuild_hash.location
with open(
@@ -83,9 +75,6 @@ class EbuildMetadataPhase(SubProcess):
settings.setcpv(self.cpv)
settings.configdict["pkg"]["EAPI"] = parsed_eapi
-# This requires above setcpv and EAPI setup.
-await _setup_locale(self.settings)
-
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
@@ -150,16 +139,6 @@ class EbuildMetadataPhase(SubProcess):
self._proc = retval
-def _async_start_done(self, future):
-future.cancelled() or future.result()
-if future.cancelled():
-self.cancel()
-self._was_cancelled()
-
-if self.returncode is not None:
-self._unregister()
-self.wait()
-
def _output_handler(self):
while True:
buf = self._read_buf(self._files.ebuild)
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index c8caf73722..c81bf54a81 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2024 Gentoo Authors
+# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -24,7 +24,6 @@ from portage.package.ebuild.prepare_build_dirs import (
_prepare_fake_distdir,
_prepare_fake_filesdir,
)
-from portage.eapi import _get_eapi_attrs
from portage.util import writemsg, ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -55,34 +54,12 @@ portage.proxy.lazyimport.lazyimport(
+ "_post_src_install_write_metadata,"
+ "_preinst_bsdflags",
"portage.util.futures.unix_events:_set_nonblocking",
-"portage.util.locale:async_check_locale,split_LC_ALL",
)
from portage import os
from portage import _encodings
from portage import _unicode_encode
-async def _setup_locale(settings):
-eapi_attrs = _get_eapi_attrs(settings["EAPI"])
-if eapi_attrs.posixish_locale:
-split_LC_ALL(settings)
-settings["LC_COLLATE"] = "C"
-# check_locale() returns None when check can not be executed.
-if await async_check_locale(silent=True, env=settings.environ()) is
False:
-# try another locale
-for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
-settings["LC_CTYPE"] = l
-if await async_check_locale(silent=True,
env=settings.environ()):
-# TODO: output the following only once
-# writemsg(
-# _("!!! LC_CTYPE unsupported, using %s instead\n")
-# % self.settings["LC_CTYPE"]
-# )
-break
-else:
-raise AssertionError("C locale did not pass the test!")
-
-
class EbuildPhase(CompositeTask):
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 419cce79f9082308c848df0a98f367de4d1c50a3
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 11 21:58:10 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 12 07:56:10 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=419cce79
process._exec: Use _start_fork for os.fork() error handling
Use _start_fork for os.fork() error handling, ensuring
that if exec fails then the child process will display
a traceback before it exits via os._exit to suppress any
finally blocks from parent's call stack (bug 345289).
Bug: https://bugs.gentoo.org/345289
Bug: https://bugs.gentoo.org/916566
Bug: https://bugs.gentoo.org/924313
Signed-off-by: Zac Medico gentoo.org>
lib/portage/process.py | 259 -
1 file changed, 151 insertions(+), 108 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 6cad250e34..f4758c824c 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -961,7 +961,13 @@ def _exec(
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
-try:
+# unshare() may not be supported by libc
+if not hasattr(libc, "unshare"):
+unshare_net = False
+unshare_ipc = False
+unshare_mount = False
+unshare_pid = False
+else:
# Since a failed unshare call could corrupt process
# state, first validate that the call can succeed.
# The parent process should call _unshare_validate
@@ -992,117 +998,154 @@ def _exec(
)
else:
if unshare_pid:
-main_child_pid = os.fork()
-if main_child_pid == 0:
-# pid namespace requires us to become init
-binary, myargs = (
-portage._python_interpreter,
-[
-portage._python_interpreter,
-os.path.join(portage._bin_path,
"pid-ns-init"),
-_unicode_encode(
-"" if uid is None else str(uid)
-),
-_unicode_encode(
-"" if gid is None else str(gid)
-),
-_unicode_encode(
-""
-if groups is None
-else ",".join(
-str(group) for group in groups
-)
-),
-_unicode_encode(
-"" if umask is None else str(umask)
-),
-_unicode_encode(
-",".join(str(fd) for fd in
fd_pipes)
-),
-binary,
-]
-+ myargs,
-)
-uid = None
-gid = None
-groups = None
-umask = None
-else:
-# Execute a supervisor process which will
forward
-# signals to init and forward exit status to
the
-# parent process. The supervisor process runs
in
-# the global pid namespace, so skip /proc
remount
-# and other setup that's intended only for the
-# init process.
-binary, myargs = portage._python_interpreter, [
+# pid namespace requires us to become init
+binary, myargs = (
+portage._python_interpreter,
+[
portage._python_interpreter,
os.path.join(portage._bin_path,
"pid-ns-init"),
-str(main_child_pid),
+_unicode_encode("" if uid is None else
str(uid)),
+_unicode_encode("" if gid is None else
str(gid)),
+
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: a1024a6d02ca3a55f86525e0d8d5089e754d3713
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 11 05:38:58 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 11 19:46:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a1024a6d
spawn_wrapper: Make pre_exec function picklable
Local functions are unpicklable, which triggered this error
with the multiprocessing "spawn" start method:
AttributeError: Can't pickle local object
'spawn_wrapper.__call__.._pre_exec'
Bug: https://bugs.gentoo.org/924273
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_selinux.py | 17 -
lib/portage/process.py | 26 ++
2 files changed, 26 insertions(+), 17 deletions(-)
diff --git a/lib/portage/_selinux.py b/lib/portage/_selinux.py
index bf6ad24895..5ae1b4e715 100644
--- a/lib/portage/_selinux.py
+++ b/lib/portage/_selinux.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# Don't use the unicode-wrapped os and shutil modules here since
@@ -6,6 +6,7 @@
import os
import shutil
import warnings
+from functools import partial
try:
import selinux
@@ -134,14 +135,12 @@ class spawn_wrapper:
def __call__(self, *args, **kwargs):
if self._con is not None:
-pre_exec = kwargs.get("pre_exec")
-
-def _pre_exec():
-if pre_exec is not None:
-pre_exec()
-setexec(self._con)
-
-kwargs["pre_exec"] = _pre_exec
+pre_exec = partial(setexec, self._con)
+kwargs["pre_exec"] = (
+portage.process._chain_pre_exec_fns(pre_exec,
kwargs["pre_exec"])
+if kwargs.get("pre_exec")
+else pre_exec
+)
return self._spawn_func(*args, **kwargs)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index a33e7b4747..6cad250e34 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -18,7 +18,7 @@ import os as _os
import warnings
from dataclasses import dataclass
-from functools import lru_cache
+from functools import lru_cache, partial
from typing import Any, Optional, Callable, Union
from portage import os
@@ -1383,18 +1383,28 @@ def _start_fork(
return pid
-class _setup_pipes_after_fork:
-def __init__(self, target, fd_pipes):
+class _chain_pre_exec_fns:
+"""
+Wraps a target function to call pre_exec functions just before
+the original target function.
+"""
+
+def __init__(self, target, *args):
self._target = target
-self._fd_pipes = fd_pipes
+self._pre_exec_fns = args
def __call__(self, *args, **kwargs):
-for fd in set(self._fd_pipes.values()):
-os.set_inheritable(fd, True)
-_setup_pipes(self._fd_pipes, close_fds=False, inheritable=True)
+for pre_exec in self._pre_exec_fns:
+pre_exec()
return self._target(*args, **kwargs)
+def _setup_pipes_after_fork(fd_pipes):
+for fd in set(fd_pipes.values()):
+os.set_inheritable(fd, True)
+_setup_pipes(fd_pipes, close_fds=False, inheritable=True)
+
+
def _start_proc(
target: Callable[..., None],
args: Optional[tuple[Any, ...]] = (),
@@ -1419,7 +1429,7 @@ def _start_proc(
# which ForkProcess does not handle because its target
# function does not necessarily exec.
if fd_pipes and multiprocessing.get_start_method() == "fork":
-target = _setup_pipes_after_fork(target, fd_pipes)
+target = _chain_pre_exec_fns(target, partial(_setup_pipes_after_fork,
fd_pipes))
fd_pipes = None
proc = ForkProcess(
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dep/, lib/portage/dep/
commit: 12f6056da88041f82a9c9dfc23ee0eab39077782
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 11 04:12:45 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 11 04:36:22 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=12f6056d
_overlap_dnf: deduplicate any-of blocks
Duplicate any-of blocks are eliminated since DNF expansion
of duplicates is nonsensical.
Bug: https://bugs.gentoo.org/891137
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dep/dep_check.py | 17 ++---
lib/portage/tests/dep/test_overlap_dnf.py | 31 +--
2 files changed, 39 insertions(+), 9 deletions(-)
diff --git a/lib/portage/dep/dep_check.py b/lib/portage/dep/dep_check.py
index 5ca0995a87..c361ee59e2 100644
--- a/lib/portage/dep/dep_check.py
+++ b/lib/portage/dep/dep_check.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["dep_check", "dep_eval", "dep_wordreduce", "dep_zapdeps"]
@@ -963,7 +963,8 @@ def _overlap_dnf(dep_struct):
order to minimize the number of packages chosen to satisfy cases like
"|| ( foo bar ) || ( bar baz )" as in bug #632026. Non-overlapping
groups are excluded from the conversion, since DNF leads to exponential
-explosion of the formula.
+explosion of the formula. Duplicate || groups are eliminated since
+DNF expansion of duplicates is nonsensical (bug #891137).
When dep_struct does not contain any overlapping groups, no DNF
conversion will be performed, and dep_struct will be returned as-is.
@@ -1021,7 +1022,17 @@ def _overlap_dnf(dep_struct):
if len(disjunctions) > 1:
overlap = True
# convert overlapping disjunctions to DNF
-result.extend(_dnf_convert(sorted(disjunctions.values(),
key=order_key)))
+dedup_set = set()
+unique_disjunctions = []
+for x in sorted(disjunctions.values(), key=order_key):
+dep_repr = portage.dep.paren_enclose(x, opconvert=True)
+if dep_repr not in dedup_set:
+dedup_set.add(dep_repr)
+unique_disjunctions.append(x)
+if len(unique_disjunctions) > 1:
+result.extend(_dnf_convert(unique_disjunctions))
+else:
+result.extend(unique_disjunctions)
else:
# pass through non-overlapping disjunctions
result.append(disjunctions.popitem()[1])
diff --git a/lib/portage/tests/dep/test_overlap_dnf.py
b/lib/portage/tests/dep/test_overlap_dnf.py
index 77352021a9..7fd1cfe7dd 100644
--- a/lib/portage/tests/dep/test_overlap_dnf.py
+++ b/lib/portage/tests/dep/test_overlap_dnf.py
@@ -1,4 +1,4 @@
-# Copyright 2017-2023 Gentoo Authors
+# Copyright 2017-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -51,22 +51,41 @@ class OverlapDNFTestCase(TestCase):
class DuplicateOverlapDNFTestCase(TestCase):
def testDuplicateOverlapDNF(self):
"""
-Demonstrate unnecessary DNF expansion for duplicate
-any-of blocks as in bug 891137.
+Demonstrate deduplication of any-of blocks, preventing unnecessary
+DNF expansion for duplicate any-of blocks as in bug 891137.
"""
test_cases = (
+("|| ( cat/A cat/B ) || ( cat/A cat/B )", [["||", "cat/A",
"cat/B"]]),
(
-"|| ( cat/A cat/B ) || ( cat/A cat/B )",
+"|| ( cat/A cat/B ) cat/E || ( cat/C cat/D ) || ( cat/A cat/B
)",
+["cat/E", ["||", "cat/A", "cat/B"], ["||", "cat/C", "cat/D"]],
+),
+(
+"|| ( cat/A cat/B ) cat/D || ( cat/B cat/C ) || ( cat/A cat/B
)",
[
+"cat/D",
[
"||",
-["cat/A", "cat/A"],
["cat/A", "cat/B"],
-["cat/B", "cat/A"],
+["cat/A", "cat/C"],
["cat/B", "cat/B"],
+["cat/B", "cat/C"],
],
],
),
+(
+"|| ( cat/A cat/B ) || ( cat/C cat/D ) || ( ( cat/B cat/E )
cat/F ) || ( cat/A cat/B )",
+[
+[
+"||",
+["cat/A", "cat/B", "cat/E"],
+["cat/A", "cat/F"],
+["cat/B", "cat/B", "cat/E"],
+["cat/B", "cat/F"],
+],
+["||", "cat/C", "cat/D"],
+],
+),
)
for dep_str, result in test_cases:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/
commit: e02feddd148f4a4a5854edd81b9aa67844d8956f
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 11 04:31:41 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 11 04:32:27 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e02feddd
test_gpkg_metadata_url_case: fix string format for pyupgrade
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
index 3773049227..e9f4111279 100644
--- a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
+++ b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
@@ -77,7 +77,7 @@ class test_gpkg_metadata_url_case(TestCase):
test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
meta_from_url = test_gpkg.get_metadata_url(
-"http://{0}:{1}/test.gpkg.tar".format(*server.server_address)
+"http://{}:{}/test.gpkg.tar".format(*server.server_address)
)
self.assertEqual(meta, meta_from_url)
@@ -148,7 +148,7 @@
IkCfAP49AOYjzuQPP0n5P0SGCINnAVEXN7QLQ4PurY/lt7cT2gEAq01stXjFhrz5
self.assertRaises(
InvalidSignature,
test_gpkg.get_metadata_url,
-
"http://{0}:{1}/test-2.gpkg.tar".format(*server.server_address),
+"http://{}:{}/test-2.gpkg.tar".format(*server.server_address),
)
finally:
if gpg is not None:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dep/
commit: 15c173dcea2401a13cfb3313918c77d7dbde133d Author: Zac Medico gentoo org> AuthorDate: Sun Feb 11 03:28:27 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Feb 11 03:29:23 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=15c173dc DuplicateOverlapDNFTestCase: Add test for bug 891137 Bug: https://bugs.gentoo.org/891137 Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/dep/test_overlap_dnf.py | 30 +- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/portage/tests/dep/test_overlap_dnf.py b/lib/portage/tests/dep/test_overlap_dnf.py index dfeded3b40..77352021a9 100644 --- a/lib/portage/tests/dep/test_overlap_dnf.py +++ b/lib/portage/tests/dep/test_overlap_dnf.py @@ -1,4 +1,4 @@ -# Copyright 2017 Gentoo Foundation +# Copyright 2017-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -46,3 +46,31 @@ class OverlapDNFTestCase(TestCase): _overlap_dnf(use_reduce(dep_str, token_class=Atom, opconvert=True)), result, ) + + +class DuplicateOverlapDNFTestCase(TestCase): +def testDuplicateOverlapDNF(self): +""" +Demonstrate unnecessary DNF expansion for duplicate +any-of blocks as in bug 891137. +""" +test_cases = ( +( +"|| ( cat/A cat/B ) || ( cat/A cat/B )", +[ +[ +"||", +["cat/A", "cat/A"], +["cat/A", "cat/B"], +["cat/B", "cat/A"], +["cat/B", "cat/B"], +], +], +), +) + +for dep_str, result in test_cases: +self.assertEqual( +_overlap_dnf(use_reduce(dep_str, token_class=Atom, opconvert=True)), +result, +)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/
commit: 9100b425a38f0f0631a4377da82f421ad87546b4 Author: Zac Medico gentoo org> AuthorDate: Sat Feb 10 21:01:17 2024 + Commit: Zac Medico gentoo org> CommitDate: Sat Feb 10 21:02:13 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9100b425 test_gpkg_metadata_url_case: fix pylint W0611: Unused import random (unused-import) Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py index 271d3e4d5f..3773049227 100644 --- a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py +++ b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py @@ -2,7 +2,6 @@ # Portage Unit Testing Functionality import io -import random import tarfile import tempfile from functools import partial
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/
commit: d279a3a5c4907f2ed8c7dc52d9c240ee33a35987
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 10 20:41:01 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 10 20:50:38 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d279a3a5
test_gpkg_metadata_url_case: optimize httpd port allocation
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 28 +++-
1 file changed, 8 insertions(+), 20 deletions(-)
diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
index 422ca7d3fa..271d3e4d5f 100644
--- a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
+++ b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
@@ -19,7 +19,7 @@ from portage.gpg import GPG
class test_gpkg_metadata_url_case(TestCase):
-def httpd(self, directory, port, httpd_future):
+def httpd(self, directory, httpd_future):
try:
import http.server
import socketserver
@@ -28,11 +28,11 @@ class test_gpkg_metadata_url_case(TestCase):
Handler = partial(http.server.SimpleHTTPRequestHandler,
directory=directory)
-with socketserver.TCPServer(("127.0.0.1", port), Handler) as httpd:
+with socketserver.TCPServer(("127.0.0.1", 0), Handler) as httpd:
httpd_future.set_result(httpd)
httpd.serve_forever()
-def start_http_server(self, directory, port):
+def start_http_server(self, directory):
try:
import threading
except ImportError:
@@ -40,7 +40,7 @@ class test_gpkg_metadata_url_case(TestCase):
httpd_future = Future()
server = threading.Thread(
-target=self.httpd, args=(directory, port, httpd_future),
daemon=True
+target=self.httpd, args=(directory, httpd_future), daemon=True
)
server.start()
return httpd_future.result()
@@ -59,13 +59,7 @@ class test_gpkg_metadata_url_case(TestCase):
server = None
try:
settings = playground.settings
-for _ in range(0, 5):
-port = random.randint(3, 6)
-try:
-server = self.start_http_server(tmpdir, port)
-except OSError:
-continue
-break
+server = self.start_http_server(tmpdir)
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
@@ -84,7 +78,7 @@ class test_gpkg_metadata_url_case(TestCase):
test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
meta_from_url = test_gpkg.get_metadata_url(
-"http://127.0.0.1:; + str(port) + "/test.gpkg.tar"
+"http://{0}:{1}/test.gpkg.tar".format(*server.server_address)
)
self.assertEqual(meta, meta_from_url)
@@ -111,13 +105,7 @@ class test_gpkg_metadata_url_case(TestCase):
gpg = GPG(settings)
gpg.unlock()
-for _ in range(0, 5):
-port = random.randint(3, 6)
-try:
-server = self.start_http_server(tmpdir, port)
-except OSError:
-continue
-break
+server = self.start_http_server(tmpdir)
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
@@ -161,7 +149,7 @@
IkCfAP49AOYjzuQPP0n5P0SGCINnAVEXN7QLQ4PurY/lt7cT2gEAq01stXjFhrz5
self.assertRaises(
InvalidSignature,
test_gpkg.get_metadata_url,
-"http://127.0.0.1:; + str(port) + "/test-2.gpkg.tar",
+
"http://{0}:{1}/test-2.gpkg.tar".format(*server.server_address),
)
finally:
if gpg is not None:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/
commit: d320211266b95dc8cbea295cb234a607246d955f
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 10 20:25:28 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 10 20:26:17 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d3202112
test_gpkg_metadata_url_case: shutdown http server daemon threads
Bug: https://bugs.gentoo.org/924192
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 15 ---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
index 857defe188..422ca7d3fa 100644
--- a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
+++ b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
@@ -7,6 +7,7 @@ import tarfile
import tempfile
from functools import partial
from os import urandom
+from concurrent.futures import Future
from portage.gpkg import gpkg
from portage import os
@@ -18,7 +19,7 @@ from portage.gpg import GPG
class test_gpkg_metadata_url_case(TestCase):
-def httpd(self, directory, port):
+def httpd(self, directory, port, httpd_future):
try:
import http.server
import socketserver
@@ -28,6 +29,7 @@ class test_gpkg_metadata_url_case(TestCase):
Handler = partial(http.server.SimpleHTTPRequestHandler,
directory=directory)
with socketserver.TCPServer(("127.0.0.1", port), Handler) as httpd:
+httpd_future.set_result(httpd)
httpd.serve_forever()
def start_http_server(self, directory, port):
@@ -36,11 +38,12 @@ class test_gpkg_metadata_url_case(TestCase):
except ImportError:
self.skipTest("threading module not exists")
+httpd_future = Future()
server = threading.Thread(
-target=self.httpd, args=(directory, port), daemon=True
+target=self.httpd, args=(directory, port, httpd_future),
daemon=True
)
server.start()
-return server
+return httpd_future.result()
def test_gpkg_get_metadata_url(self):
playground = ResolverPlayground(
@@ -53,6 +56,7 @@ class test_gpkg_metadata_url_case(TestCase):
}
)
tmpdir = tempfile.mkdtemp()
+server = None
try:
settings = playground.settings
for _ in range(0, 5):
@@ -85,6 +89,8 @@ class test_gpkg_metadata_url_case(TestCase):
self.assertEqual(meta, meta_from_url)
finally:
+if server is not None:
+server.shutdown()
shutil.rmtree(tmpdir)
playground.cleanup()
@@ -99,6 +105,7 @@ class test_gpkg_metadata_url_case(TestCase):
)
tmpdir = tempfile.mkdtemp()
gpg = None
+server = None
try:
settings = playground.settings
gpg = GPG(settings)
@@ -159,5 +166,7 @@
IkCfAP49AOYjzuQPP0n5P0SGCINnAVEXN7QLQ4PurY/lt7cT2gEAq01stXjFhrz5
finally:
if gpg is not None:
gpg.stop()
+if server is not None:
+server.shutdown()
shutil.rmtree(tmpdir)
playground.cleanup()
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 03be12ec5f46629fa928e5fcd45d3fe6745d5d0a
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 22:12:02 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 10 06:08:59 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=03be12ec
GPG: Use threading.Event for thread safety
Use threading.Event for thread safety during GPG stop, and use the
wait method to improve responsiveness for stop requests.
Bug: https://bugs.gentoo.org/924192
Signed-off-by: Zac Medico gentoo.org>
lib/portage/gpg.py | 14 --
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/lib/portage/gpg.py b/lib/portage/gpg.py
index 3067872244..d8a4cfcfc4 100644
--- a/lib/portage/gpg.py
+++ b/lib/portage/gpg.py
@@ -1,10 +1,9 @@
-# Copyright 2001-2020 Gentoo Authors
+# Copyright 2001-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import subprocess
import sys
import threading
-import time
from portage import os
from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
@@ -24,6 +23,7 @@ class GPG:
"""
self.settings = settings
self.thread = None
+self._terminated = None
self.GPG_signing_base_command = self.settings.get(
"BINPKG_GPG_SIGNING_BASE_COMMAND"
)
@@ -73,6 +73,7 @@ class GPG:
self.GPG_unlock_command = shlex_split(
varexpand(self.GPG_unlock_command, mydict=self.settings)
)
+self._terminated = threading.Event()
self.thread = threading.Thread(target=self.gpg_keepalive,
daemon=True)
self.thread.start()
@@ -81,16 +82,17 @@ class GPG:
Stop keepalive thread.
"""
if self.thread is not None:
-self.keepalive = False
+self._terminated.set()
def gpg_keepalive(self):
"""
Call GPG unlock command every 5 mins to avoid the passphrase expired.
"""
count = 0
-while self.keepalive:
+while not self._terminated.is_set():
if count < 5:
-time.sleep(60)
+if self._terminated.wait(60):
+break
count += 1
continue
else:
@@ -102,5 +104,5 @@ class GPG:
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
)
-if proc.wait() != os.EX_OK:
+if proc.wait() != os.EX_OK and not self._terminated.is_set():
raise GPGException("GPG keepalive failed")
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/, bin/, lib/_emerge/
commit: d7115d18dada572b6b10d6be30d0fa7fb325a2c8
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 17:19:54 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 10 06:08:58 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d7115d18
GPG: Proactively stop to avoid "GPG keepalive failed" error in pypy ci jobs
This seems to help mitigate pypy ci job hangs like those fba76a545f2
triggered.
Bug: https://bugs.gentoo.org/924192
Signed-off-by: Zac Medico gentoo.org>
bin/quickpkg | 13 -
lib/_emerge/actions.py | 10 +++---
lib/portage/tests/gpkg/test_gpkg_gpg.py | 23 ++-
lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 5 -
4 files changed, 41 insertions(+), 10 deletions(-)
diff --git a/bin/quickpkg b/bin/quickpkg
index 8443a00e64..c688c5312e 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import argparse
@@ -341,10 +341,6 @@ def quickpkg_main(options, args, eout):
portage.settings.features.remove("xattr")
portage.settings.lock()
-if portage.settings.get("BINPKG_GPG_SIGNING_KEY", None):
-gpg = GPG(portage.settings)
-gpg.unlock()
-
infos = {}
infos["successes"] = []
infos["missing"] = []
@@ -444,11 +440,18 @@ if __name__ == "__main__":
def sigwinch_handler(signum, frame):
lines, eout.term_columns = portage.output.get_term_size()
+gpg = None
+if portage.settings.get("BINPKG_GPG_SIGNING_KEY", None):
+gpg = GPG(portage.settings)
+gpg.unlock()
+
signal.signal(signal.SIGWINCH, sigwinch_handler)
try:
retval = quickpkg_main(options, args, eout)
finally:
os.umask(old_umask)
signal.signal(signal.SIGWINCH, signal.SIG_DFL)
+if gpg is not None:
+gpg.stop()
global_event_loop().close()
sys.exit(retval)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 2710c4856c..d36a799244 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import collections
@@ -548,8 +548,10 @@ def action_build(
mergelist_shown = True
if retval != os.EX_OK:
return retval
+return os.EX_OK
-else:
+gpg = None
+try:
if not mergelist_shown:
# If we haven't already shown the merge list above, at
# least show warnings about missed updates and such.
@@ -688,8 +690,10 @@ def action_build(
ldpath_mtimes,
autoclean=1,
)
-
return retval
+finally:
+if gpg is not None:
+gpg.stop()
def action_config(settings, trees, myopts, myfiles):
diff --git a/lib/portage/tests/gpkg/test_gpkg_gpg.py
b/lib/portage/tests/gpkg/test_gpkg_gpg.py
index a2dc92150b..d7eae4a82b 100644
--- a/lib/portage/tests/gpkg/test_gpkg_gpg.py
+++ b/lib/portage/tests/gpkg/test_gpkg_gpg.py
@@ -1,4 +1,4 @@
-# Copyright Gentoo Foundation 2006-2020
+# Copyright 2022-2024 Gentoo Authors
# Portage Unit Testing Functionality
import io
@@ -26,6 +26,7 @@ class test_gpkg_gpg_case(TestCase):
}
)
tmpdir = tempfile.mkdtemp()
+gpg = None
try:
settings = playground.settings
@@ -68,6 +69,8 @@ class test_gpkg_gpg_case(TestCase):
InvalidSignature, binpkg_2.decompress, os.path.join(tmpdir,
"test")
)
finally:
+if gpg is not None:
+gpg.stop()
shutil.rmtree(tmpdir)
playground.cleanup()
@@ -81,6 +84,7 @@ class test_gpkg_gpg_case(TestCase):
}
)
tmpdir = tempfile.mkdtemp()
+gpg = None
try:
settings = playground.settings
@@ -112,6 +116,8 @@ class test_gpkg_gpg_case(TestCase):
)
finally:
+if gpg is not None:
+gpg.stop()
shutil.rmtree(tmpdir)
playground.cleanup()
@@ -133,6 +139,7 @@ class test_gpkg_gpg_case(TestCase):
}
)
tmpdir = tempfile.mkdtemp()
+gpg = None
try:
settings = playground.settings
@@ -151,6 +158,8 @@ class test_gpkg_gpg_case(TestCase):
binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir,
"test-1.gpkg.tar"))
binpkg_2.decompress(os.path.join(tmpdir, "test"))
finally:
+if gpg is not None:
+gpg.stop()
shutil.rmtree(tmpdir)
playground.cleanup()
@@ -165,6
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: ad61940b03be2f24c0b54c1070a4923abe18e633
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 16:22:45 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 9 23:52:11 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ad61940b
gpkg: Less aggressive subprocess.Popen kill in order to avoid BrokenPipeError
Do not kill tar_stream_reader instances if we can successfully
close them, since that can trigger BrokenPipeError during
self.proc.stdin.close() calls, and this state is best avoided
because it's unclear how the caller should handle the error.
If a BrokenPipeError does occur then simply print a traceback
and hope that the corresponding file descriptor is closed
during garbage collection. Do not try to reverse the order
of self.proc.kill() and self.proc.stdin.close() as in commit
fba76a545f2 since that triggered pypy ci job hangs for which
no reliable solution has been found so far.
Bug: https://bugs.gentoo.org/923368
Signed-off-by: Zac Medico gentoo.org>
lib/portage/gpkg.py | 19 ++-
1 file changed, 14 insertions(+), 5 deletions(-)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 031b3f87cb..f1d8f97f8e 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -1,7 +1,8 @@
-# Copyright 2001-2020 Gentoo Authors
+# Copyright 2001-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import tarfile
+import traceback
import io
import threading
import subprocess
@@ -151,7 +152,10 @@ class tar_stream_writer:
if self.proc is not None:
self.killed = True
self.proc.kill()
-self.proc.stdin.close()
+try:
+self.proc.stdin.close()
+except BrokenPipeError:
+traceback.print_exc()
self.close()
def _cmd_read_thread(self):
@@ -213,7 +217,7 @@ class tar_stream_writer:
if self.proc is not None:
self.proc.stdin.close()
if self.proc.wait() != os.EX_OK:
-if not self.error:
+if not (self.killed or self.error):
raise CompressorOperationFailed("compression failed")
if self.read_thread.is_alive():
self.read_thread.join()
@@ -349,7 +353,10 @@ class tar_stream_reader:
if self.proc is not None:
self.killed = True
self.proc.kill()
-self.proc.stdin.close()
+try:
+self.proc.stdin.close()
+except BrokenPipeError:
+traceback.print_exc()
self.close()
def read(self, bufsize=-1):
@@ -986,11 +993,13 @@ class gpkg:
try:
image_safe = tar_safe_extract(image, "image")
image_safe.extractall(decompress_dir)
+image_tar.close()
except Exception as ex:
writemsg(colorize("BAD", "!!!Extract failed."))
raise
finally:
-image_tar.kill()
+if not image_tar.closed:
+image_tar.kill()
def update_metadata(self, metadata, new_basename=None, force=False):
"""
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/
commit: 3dac4f892479d6215c378f761505ab3d41a4b3ef Author: Zac Medico gentoo org> AuthorDate: Fri Feb 9 23:18:11 2024 + Commit: Zac Medico gentoo org> CommitDate: Fri Feb 9 23:18:26 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3dac4f89 test_gpkg_path_case: Add missing playground cleanup Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/gpkg/test_gpkg_path.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/portage/tests/gpkg/test_gpkg_path.py b/lib/portage/tests/gpkg/test_gpkg_path.py index fc57135949..19451e2e9b 100644 --- a/lib/portage/tests/gpkg/test_gpkg_path.py +++ b/lib/portage/tests/gpkg/test_gpkg_path.py @@ -1,4 +1,4 @@ -# Copyright Gentoo Foundation 2006 +# Copyright 2022-2024 Gentoo Authors # Portage Unit Testing Functionality import tempfile @@ -308,6 +308,7 @@ class test_gpkg_path_case(TestCase): self.assertEqual(r, ()) finally: shutil.rmtree(tmpdir) +playground.cleanup() def test_gpkg_long_filename(self): playground = ResolverPlayground(
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/
commit: 8dd69569b968ca0193b131c393e70855015a50dd
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 22:19:22 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 9 22:19:35 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8dd69569
ManifestTestCase: Fix tempdir removal
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_manifest.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/portage/tests/util/test_manifest.py
b/lib/portage/tests/util/test_manifest.py
index 3412d568d7..2d41b9fc97 100644
--- a/lib/portage/tests/util/test_manifest.py
+++ b/lib/portage/tests/util/test_manifest.py
@@ -11,7 +11,8 @@ from portage.tests import TestCase
class ManifestTestCase(TestCase):
def test_simple_addFile(self):
-tempdir = Path(tempfile.mkdtemp()) / "app-portage" / "diffball"
+base_tempdir = tempfile.mkdtemp()
+tempdir = Path(base_tempdir) / "app-portage" / "diffball"
manifest = Manifest(str(tempdir), required_hashes=["SHA512",
"BLAKE2B"])
(tempdir / "files").mkdir(parents=True)
@@ -30,4 +31,4 @@ class ManifestTestCase(TestCase):
manifest.getFileData("AUX", "test.patch", "SHA512"),
"e30d069dcf284cbcb2d5685f03ca362469026b469dec4f8655d0c9a2bf317f5d9f68f61855ea403f4959bc0b9c003ae824fb9d6ab2472a739950623523af9da9",
)
-shutil.rmtree(str(tempdir))
+shutil.rmtree(base_tempdir)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/resolver/
commit: be37f0761752f13a855aed66fa6e49e2f7211a0f Author: Zac Medico gentoo org> AuthorDate: Fri Feb 9 21:36:02 2024 + Commit: Zac Medico gentoo org> CommitDate: Fri Feb 9 21:36:28 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=be37f076 EAPITestCase: Disable playground debug so tempdir is cleaned up Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/resolver/test_eapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/tests/resolver/test_eapi.py b/lib/portage/tests/resolver/test_eapi.py index 5d425ccdb9..32dcb49895 100644 --- a/lib/portage/tests/resolver/test_eapi.py +++ b/lib/portage/tests/resolver/test_eapi.py @@ -1,4 +1,4 @@ -# Copyright 2010-2020 Gentoo Authors +# Copyright 2010-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -199,7 +199,7 @@ class EAPITestCase(TestCase): mergelist=["dev-libs/A-1.0", "dev-libs/B-1.0"], ) -playground = ResolverPlayground(ebuilds=ebuilds, debug=True) +playground = ResolverPlayground(ebuilds=ebuilds) try: playground.run_TestCase(test_case) self.assertEqual(test_case.test_success, True, test_case.fail_msg)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/glsa/
commit: 9cbc53ad6c483500c949c1acd70c6cbb2d7cee86 Author: Zac Medico gentoo org> AuthorDate: Fri Feb 9 21:38:22 2024 + Commit: Zac Medico gentoo org> CommitDate: Fri Feb 9 21:38:39 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9cbc53ad SecuritySetTestCase: Disable playground debug so tempdir is cleaned up Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/glsa/test_security_set.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/tests/glsa/test_security_set.py b/lib/portage/tests/glsa/test_security_set.py index a0ba1e5b45..1206d9f80f 100644 --- a/lib/portage/tests/glsa/test_security_set.py +++ b/lib/portage/tests/glsa/test_security_set.py @@ -1,4 +1,4 @@ -# Copyright 2013-2023 Gentoo Authors +# Copyright 2013-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 @@ -226,7 +226,7 @@ class SecuritySetTestCase(TestCase): # Give each GLSA a clean slate for glsa in glsas: playground = ResolverPlayground( -ebuilds=ebuilds, installed=installed, world=world, debug=True +ebuilds=ebuilds, installed=installed, world=world, debug=False ) try:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/
commit: d702c7684093dccae87df080d0b7701dddf95091
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 21:27:50 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 9 21:30:10 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d702c768
ManifestTestCase: Remove tempdir
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_manifest.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/portage/tests/util/test_manifest.py
b/lib/portage/tests/util/test_manifest.py
index 49bcbc1a54..3412d568d7 100644
--- a/lib/portage/tests/util/test_manifest.py
+++ b/lib/portage/tests/util/test_manifest.py
@@ -1,6 +1,7 @@
-# Copyright 2022 Gentoo Authors
+# Copyright 2022-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import shutil
import tempfile
from pathlib import Path
@@ -29,3 +30,4 @@ class ManifestTestCase(TestCase):
manifest.getFileData("AUX", "test.patch", "SHA512"),
"e30d069dcf284cbcb2d5685f03ca362469026b469dec4f8655d0c9a2bf317f5d9f68f61855ea403f4959bc0b9c003ae824fb9d6ab2472a739950623523af9da9",
)
+shutil.rmtree(str(tempdir))
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/update/
commit: 091e204c2c3d66180cfe59c310291f15ef02e2bb Author: Zac Medico gentoo org> AuthorDate: Fri Feb 9 21:17:49 2024 + Commit: Zac Medico gentoo org> CommitDate: Fri Feb 9 21:29:40 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=091e204c MoveEntTestCase: Disable playground debug so tempdir is cleaned up Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/update/test_move_ent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/tests/update/test_move_ent.py b/lib/portage/tests/update/test_move_ent.py index 169b892a30..fe968f12a0 100644 --- a/lib/portage/tests/update/test_move_ent.py +++ b/lib/portage/tests/update/test_move_ent.py @@ -1,4 +1,4 @@ -# Copyright 2012-2021 Gentoo Authors +# Copyright 2012-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import sys @@ -287,7 +287,7 @@ class MoveEntTestCase(TestCase): f'FEATURES="binpkg-multi-instance pkgdir-index-trusted"', ), }, -debug=True, +debug=False, ) settings = playground.settings
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dbapi/
commit: fc6cace9fddfa3a2c5567e35156b2ee7ef39dce1
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 21:03:55 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 9 21:08:32 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fc6cace9
AuxdbTestCase: Add missing playground cleanup
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/dbapi/test_auxdb.py | 67 ---
1 file changed, 38 insertions(+), 29 deletions(-)
diff --git a/lib/portage/tests/dbapi/test_auxdb.py
b/lib/portage/tests/dbapi/test_auxdb.py
index c11eed73e8..0de0123a5f 100644
--- a/lib/portage/tests/dbapi/test_auxdb.py
+++ b/lib/portage/tests/dbapi/test_auxdb.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Gentoo Authors
+# Copyright 2020-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -64,41 +64,50 @@ class AuxdbTestCase(TestCase):
user_config={"modules": (f"portdbapi.auxdbmodule =
{auxdbmodule}",)},
)
-portdb = playground.trees[playground.eroot]["porttree"].dbapi
-metadata_keys = ["DEFINED_PHASES", "DEPEND", "EAPI", "INHERITED"]
-
-test_func = functools.partial(
-self._run_test_mod_async, ebuilds, metadata_keys, portdb
-)
-
-results = test_func()
-
-self._compare_results(
-ebuilds, eclass_defined_phases, eclass_depend, ebuild_inherited,
results
-)
+try:
+portdb = playground.trees[playground.eroot]["porttree"].dbapi
+metadata_keys = ["DEFINED_PHASES", "DEPEND", "EAPI", "INHERITED"]
-loop = asyncio._wrap_loop()
-picklable_or_fork = picklable or multiprocessing.get_start_method ==
"fork"
-if picklable_or_fork:
-results = loop.run_until_complete(
-loop.run_in_executor(ForkExecutor(), test_func)
+test_func = functools.partial(
+self._run_test_mod_async, ebuilds, metadata_keys, portdb
)
+results = test_func()
+
self._compare_results(
ebuilds, eclass_defined_phases, eclass_depend,
ebuild_inherited, results
)
-auxdb = portdb.auxdb[portdb.getRepositoryPath("test_repo")]
-cpv = next(iter(ebuilds))
-
-modify_auxdb = functools.partial(self._modify_auxdb, auxdb, cpv)
-
-if multiproc and picklable_or_fork:
-loop.run_until_complete(loop.run_in_executor(ForkExecutor(),
modify_auxdb))
-else:
-modify_auxdb()
-
-self.assertEqual(auxdb[cpv]["RESTRICT"], "test")
+loop = asyncio._wrap_loop()
+picklable_or_fork = picklable or multiprocessing.get_start_method
== "fork"
+if picklable_or_fork:
+results = loop.run_until_complete(
+loop.run_in_executor(ForkExecutor(), test_func)
+)
+
+self._compare_results(
+ebuilds,
+eclass_defined_phases,
+eclass_depend,
+ebuild_inherited,
+results,
+)
+
+auxdb = portdb.auxdb[portdb.getRepositoryPath("test_repo")]
+cpv = next(iter(ebuilds))
+
+modify_auxdb = functools.partial(self._modify_auxdb, auxdb, cpv)
+
+if multiproc and picklable_or_fork:
+loop.run_until_complete(
+loop.run_in_executor(ForkExecutor(), modify_auxdb)
+)
+else:
+modify_auxdb()
+
+self.assertEqual(auxdb[cpv]["RESTRICT"], "test")
+finally:
+playground.cleanup()
def _compare_results(
self, ebuilds, eclass_defined_phases, eclass_depend, ebuild_inherited,
results
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: c4049323b3c4c04ccc56023fa31169da58b57831 Author: Sam James gentoo org> AuthorDate: Fri Feb 9 08:47:31 2024 + Commit: Sam James gentoo org> CommitDate: Fri Feb 9 08:51:17 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c4049323 Revert "gpkg: on error, close stdin before killing external programs" This reverts commit fba76a545f29fb8b529197c25c64700ef77413ae. This seems to cause hangs in CI with pypy3. Zac mentioned this at https://github.com/gentoo/portage/pull/1246#issuecomment-1935511379 and I can reproduce it with `tox -e pypy3-test` with PyPy 7.3.15. Signed-off-by: Sam James gentoo.org> lib/portage/gpkg.py | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py index 05f1b5f2a6..031b3f87cb 100644 --- a/lib/portage/gpkg.py +++ b/lib/portage/gpkg.py @@ -149,9 +149,9 @@ class tar_stream_writer: kill external program if any error happened in python """ if self.proc is not None: -self.proc.stdin.close() -self.proc.kill() self.killed = True +self.proc.kill() +self.proc.stdin.close() self.close() def _cmd_read_thread(self): @@ -347,9 +347,9 @@ class tar_stream_reader: kill external program if any error happened in python """ if self.proc is not None: -self.proc.stdin.close() -self.proc.kill() self.killed = True +self.proc.kill() +self.proc.stdin.close() self.close() def read(self, bufsize=-1):
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: ba33bc425363977eaf549a087b2469720a79e2a4 Author: Zac Medico gentoo org> AuthorDate: Thu Feb 8 06:46:04 2024 + Commit: Zac Medico gentoo org> CommitDate: Fri Feb 9 08:19:00 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ba33bc42 check_locale: Use multiprocessing.Process instead of os.fork() Since os.fork() is unsafe in threaded processes, use multiprocessing.Process instead. This way the fork will be eliminated when the default multiprocessing start method changes to "spawn". TODO: Make async version of check_locale and call it from EbuildPhase instead of config.environ(), since it's bad to synchronously wait for the process in the main event loop thread where config.environ() tends to be called. Bug: https://bugs.gentoo.org/923841 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/locale.py | 56 +++--- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py index a620dbd544..b5da8d949b 100644 --- a/lib/portage/util/locale.py +++ b/lib/portage/util/locale.py @@ -9,7 +9,8 @@ locale. import locale import logging -import os +import multiprocessing +import sys import textwrap import traceback @@ -96,6 +97,24 @@ def _check_locale(silent): return True +def _set_and_check_locale(silent, env, mylocale): +try: +if env is not None: +try: +locale.setlocale(locale.LC_CTYPE, mylocale) +except locale.Error: +sys.exit(2) + +ret = _check_locale(silent) +if ret is None: +sys.exit(2) +else: +sys.exit(0 if ret else 1) +except Exception: +traceback.print_exc() +sys.exit(2) + + def check_locale(silent=False, env=None): """ Check whether the locale is sane. Returns True if it is, prints @@ -116,29 +135,20 @@ def check_locale(silent=False, env=None): except KeyError: pass -pid = os.fork() -if pid == 0: -try: -if env is not None: -try: -locale.setlocale(locale.LC_CTYPE, portage._native_string(mylocale)) -except locale.Error: -os._exit(2) - -ret = _check_locale(silent) -if ret is None: -os._exit(2) -else: -os._exit(0 if ret else 1) -except Exception: -traceback.print_exc() -os._exit(2) - -pid2, ret = os.waitpid(pid, 0) -assert pid == pid2 +# TODO: Make async version of check_locale and call it from +# EbuildPhase instead of config.environ(), since it's bad to +# synchronously wait for the process in the main event loop +# thread where config.environ() tends to be called. +proc = multiprocessing.Process( +target=_set_and_check_locale, +args=(silent, env, None if env is None else portage._native_string(mylocale)), +) +proc.start() +proc.join() + pyret = None -if os.WIFEXITED(ret): -ret = os.WEXITSTATUS(ret) +if proc.exitcode >= 0: +ret = proc.exitcode if ret != 2: pyret = ret == 0
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/util/, ...
commit: c95fc64abf9698263090b3ffd4a056e989dd2be1
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 06:38:41 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 9 08:19:00 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c95fc64a
EbuildPhase: async_check_locale
Change config.environ() check_locale calls to async_check_locale
calls in the EbuildPhase _async_start method in order to eliminate
synchronous waiting for child processes in the main event loop
thread.
Bug: https://bugs.gentoo.org/923841
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py| 21
lib/_emerge/EbuildPhase.py| 28 ++-
lib/portage/package/ebuild/config.py | 26 +++--
lib/portage/util/futures/_asyncio/__init__.py | 9 +
lib/portage/util/locale.py| 28 ++-
5 files changed, 87 insertions(+), 25 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index f4f685e81c..53b7ad9624 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,12 +8,14 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"_emerge.EbuildPhase:_setup_locale",
"portage.package.ebuild._metadata_invalid:eapi_invalid",
)
from portage import os
from portage import _encodings
from portage import _unicode_decode
from portage import _unicode_encode
+from portage.util.futures import asyncio
import fcntl
@@ -44,6 +46,12 @@ class EbuildMetadataPhase(SubProcess):
_files_dict = slot_dict_class(_file_names, prefix="")
def _start(self):
+asyncio.ensure_future(
+self._async_start(), loop=self.scheduler
+).add_done_callback(self._async_start_done)
+
+async def _async_start(self):
+
ebuild_path = self.ebuild_hash.location
with open(
@@ -75,6 +83,9 @@ class EbuildMetadataPhase(SubProcess):
settings.setcpv(self.cpv)
settings.configdict["pkg"]["EAPI"] = parsed_eapi
+# This requires above setcpv and EAPI setup.
+await _setup_locale(self.settings)
+
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
@@ -139,6 +150,16 @@ class EbuildMetadataPhase(SubProcess):
self._proc = retval
+def _async_start_done(self, future):
+future.cancelled() or future.result()
+if future.cancelled():
+self.cancel()
+self._was_cancelled()
+
+if self.returncode is not None:
+self._unregister()
+self.wait()
+
def _output_handler(self):
while True:
buf = self._read_buf(self._files.ebuild)
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index c81bf54a81..c8caf73722 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import (
_prepare_fake_distdir,
_prepare_fake_filesdir,
)
+from portage.eapi import _get_eapi_attrs
from portage.util import writemsg, ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport(
+ "_post_src_install_write_metadata,"
+ "_preinst_bsdflags",
"portage.util.futures.unix_events:_set_nonblocking",
+"portage.util.locale:async_check_locale,split_LC_ALL",
)
from portage import os
from portage import _encodings
from portage import _unicode_encode
+async def _setup_locale(settings):
+eapi_attrs = _get_eapi_attrs(settings["EAPI"])
+if eapi_attrs.posixish_locale:
+split_LC_ALL(settings)
+settings["LC_COLLATE"] = "C"
+# check_locale() returns None when check can not be executed.
+if await async_check_locale(silent=True, env=settings.environ()) is
False:
+# try another locale
+for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
+settings["LC_CTYPE"] = l
+if await async_check_locale(silent=True,
env=settings.environ()):
+# TODO: output the following only once
+# writemsg(
+# _("!!! LC_CTYPE unsupported, using %s instead\n")
+# % self.settings["LC_CTYPE"]
+# )
+break
+else:
+raise AssertionError("C locale did not pass the test!")
+
+
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") +
("_ebuild_lock",)
@@ -94,6 +117,9 @@ class
[gentoo-commits] proj/portage:master commit in: lib/portage/emaint/
commit: cf9dfb8274a0bcce0776b53e5d184d5a8ce2f17d Author: Steffen Winter proton me> AuthorDate: Thu Feb 8 23:29:39 2024 + Commit: Sam James gentoo org> CommitDate: Fri Feb 9 07:29:03 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cf9dfb82 emaint: Print 'usage: ' only once in help message `ArgumentParser` prepends 'usage: ' to the usage message already. Signed-off-by: Steffen Winter proton.me> Closes: https://github.com/gentoo/portage/pull/1256 Signed-off-by: Sam James gentoo.org> lib/portage/emaint/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/emaint/main.py b/lib/portage/emaint/main.py index 0c620a1a84..ad6eea359e 100644 --- a/lib/portage/emaint/main.py +++ b/lib/portage/emaint/main.py @@ -62,7 +62,7 @@ class OptionItem: def usage(module_controller): -_usage = "usage: emaint [options] COMMAND" +_usage = "emaint [options] COMMAND" desc = ( "The emaint program provides an interface to system health "
[gentoo-commits] proj/portage:master commit in: lib/portage/util/elf/, lib/portage/dep/soname/, /
commit: da6607f012572b4d13fdc6fd5ecd4a62d62a417b
Author: Matoro Mahri matoro tk>
AuthorDate: Tue Oct 31 21:00:34 2023 +
Commit: Sam James gentoo org>
CommitDate: Fri Feb 9 07:26:32 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=da6607f0
ELF: add entries for ARC machines
Signed-off-by: Matoro Mahri matoro.tk>
Closes: https://github.com/gentoo/portage/pull/1166
Signed-off-by: Sam James gentoo.org>
NEWS| 1 +
lib/portage/dep/soname/multilib_category.py | 10 ++
lib/portage/util/elf/constants.py | 5 +
3 files changed, 16 insertions(+)
diff --git a/NEWS b/NEWS
index 220a0f4f7e..396723d8a8 100644
--- a/NEWS
+++ b/NEWS
@@ -11,6 +11,7 @@ portage-3.0.62 (UNRELEASED)
Features:
* cnf: make.conf.example.arc: add for the arc arch.
+* ELF: add entries for ARC machines
portage-3.0.61 (2024-01-05)
--
diff --git a/lib/portage/dep/soname/multilib_category.py
b/lib/portage/dep/soname/multilib_category.py
index 14a9eea770..baca439fd2 100644
--- a/lib/portage/dep/soname/multilib_category.py
+++ b/lib/portage/dep/soname/multilib_category.py
@@ -52,6 +52,11 @@ from portage.util.elf.constants import (
EM_AARCH64,
EM_ALPHA,
EM_AMDGPU,
+EM_ARC,
+EM_ARC_COMPACT,
+EM_ARC_COMPACT2,
+EM_ARC_COMPACT3,
+EM_ARC_COMPACT3_64,
EM_ARM,
EM_ALTERA_NIOS2,
EM_IA_64,
@@ -80,6 +85,11 @@ _machine_prefix_map = {
EM_ALPHA: "alpha",
EM_AMDGPU: "amdgpu",
EM_ALTERA_NIOS2: "nios2",
+EM_ARC: "arc",
+EM_ARC_COMPACT: "arc",
+EM_ARC_COMPACT2: "arc",
+EM_ARC_COMPACT3: "arc",
+EM_ARC_COMPACT3_64: "arc",
EM_ARM: "arm",
EM_IA_64: "ia64",
EM_LOONGARCH: "loong",
diff --git a/lib/portage/util/elf/constants.py
b/lib/portage/util/elf/constants.py
index 022e78d776..9216a35353 100644
--- a/lib/portage/util/elf/constants.py
+++ b/lib/portage/util/elf/constants.py
@@ -31,12 +31,17 @@ EM_S390 = 22
EM_ARM = 40
EM_SH = 42
EM_SPARCV9 = 43
+EM_ARC = 45
EM_IA_64 = 50
EM_X86_64 = 62
+EM_ARC_COMPACT = 93
EM_ALTERA_NIOS2 = 113
EM_AARCH64 = 183
+EM_ARC_COMPACT2 = 195
EM_AMDGPU = 224
EM_RISCV = 243
+EM_ARC_COMPACT3_64 = 253
+EM_ARC_COMPACT3 = 255
EM_LOONGARCH = 258
EM_ALPHA = 0x9026
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: fba76a545f29fb8b529197c25c64700ef77413ae Author: Matoro Mahri matoro tk> AuthorDate: Wed Jan 31 20:25:40 2024 + Commit: Sam James gentoo org> CommitDate: Fri Feb 9 07:08:22 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fba76a54 gpkg: on error, close stdin before killing external programs Otherwise, this may trigger a BrokenPipeError. Sensitive to buffer fullness, this was observed on 64k page size system immediately after triggering E2BIG error from kernel. Signed-off-by: Matoro Mahri matoro.tk> Signed-off-by: Sam James gentoo.org> lib/portage/gpkg.py | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py index 031b3f87cb..05f1b5f2a6 100644 --- a/lib/portage/gpkg.py +++ b/lib/portage/gpkg.py @@ -149,9 +149,9 @@ class tar_stream_writer: kill external program if any error happened in python """ if self.proc is not None: -self.killed = True -self.proc.kill() self.proc.stdin.close() +self.proc.kill() +self.killed = True self.close() def _cmd_read_thread(self): @@ -347,9 +347,9 @@ class tar_stream_reader: kill external program if any error happened in python """ if self.proc is not None: -self.killed = True -self.proc.kill() self.proc.stdin.close() +self.proc.kill() +self.killed = True self.close() def read(self, bufsize=-1):
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/
commit: a308d831ceb1f1e7d0733700117cc18c8eca09c8
Author: Matoro Mahri matoro tk>
AuthorDate: Wed Jan 31 20:26:06 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Feb 9 07:08:22 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a308d831
tests: process: calculate size of E2BIG environment variable dynamically
The size of the command line required to trigger E2BIG response from the
kernel is defined as MAX_ARG_STRLEN, which is equal to 32 * PAGE_SIZE.
Therefore the minimum size required to trigger the expected error
response must be calculated dynamically based on PAGE_SIZE.
Bug: https://bugs.gentoo.org/923368
Signed-off-by: Matoro Mahri matoro.tk>
Closes: https://github.com/gentoo/portage/pull/1246
Signed-off-by: Sam James gentoo.org>
lib/portage/tests/process/test_spawn_fail_e2big.py | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/portage/tests/process/test_spawn_fail_e2big.py
b/lib/portage/tests/process/test_spawn_fail_e2big.py
index 7a00966302..abb1113fea 100644
--- a/lib/portage/tests/process/test_spawn_fail_e2big.py
+++ b/lib/portage/tests/process/test_spawn_fail_e2big.py
@@ -2,6 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
import platform
+import resource
import pytest
@@ -12,7 +13,9 @@ from portage.const import BASH_BINARY
@pytest.mark.skipif(platform.system() != "Linux", reason="not Linux")
def test_spawnE2big(capsys, tmp_path):
env = dict()
-env["VERY_LARGE_ENV_VAR"] = "X" * 1024 * 256
+# Kernel MAX_ARG_STRLEN is defined as 32 * PAGE_SIZE
+max_arg_strlen_bytes = 32 * resource.getpagesize()
+env["VERY_LARGE_ENV_VAR"] = "X" * max_arg_strlen_bytes
logfile = tmp_path / "logfile"
echo_output = "Should never appear"
@@ -24,7 +27,7 @@ def test_spawnE2big(capsys, tmp_path):
with open(logfile) as f:
logfile_content = f.read()
assert (
-"Largest environment variable: VERY_LARGE_ENV_VAR (262164 bytes)"
+f"Largest environment variable: VERY_LARGE_ENV_VAR
({max_arg_strlen_bytes + 20} bytes)"
in logfile_content
)
assert retval == 1
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/locks/
commit: ca8d46112c5b0f7ab9f298915b1cee5a39523173
Author: Zac Medico gentoo org>
AuthorDate: Wed Feb 7 04:33:16 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 7 15:31:30 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ca8d4611
LockNonblockTestCase: Use multiprocessing.Process instead of os.fork()
Since os.fork() is unsafe in threaded processes, use
multiprocessing.Process instead. This way the fork will
be eliminated when the default multiprocessing start
method changes to "spawn".
Bug: https://bugs.gentoo.org/914876
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/locks/test_lock_nonblock.py | 52 +--
1 file changed, 26 insertions(+), 26 deletions(-)
diff --git a/lib/portage/tests/locks/test_lock_nonblock.py
b/lib/portage/tests/locks/test_lock_nonblock.py
index 9bb91b428e..d30dfe113b 100644
--- a/lib/portage/tests/locks/test_lock_nonblock.py
+++ b/lib/portage/tests/locks/test_lock_nonblock.py
@@ -1,6 +1,8 @@
# Copyright 2011-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import multiprocessing
+import sys
import tempfile
import traceback
@@ -17,37 +19,35 @@ class LockNonblockTestCase(TestCase):
try:
path = os.path.join(tempdir, "lock_me")
lock1 = portage.locks.lockfile(path)
-pid = os.fork()
-if pid == 0:
-portage.locks._close_fds()
-# Disable close_fds since we don't exec
-# (see _setup_pipes docstring).
-portage.process._setup_pipes({0: 0, 1: 1, 2: 2},
close_fds=False)
-rval = 2
-try:
-try:
-lock2 = portage.locks.lockfile(path,
flags=os.O_NONBLOCK)
-except portage.exception.TryAgain:
-rval = os.EX_OK
-else:
-rval = 1
-portage.locks.unlockfile(lock2)
-except SystemExit:
-raise
-except:
-traceback.print_exc()
-finally:
-os._exit(rval)
-
-self.assertEqual(pid > 0, True)
-pid, status = os.waitpid(pid, 0)
-self.assertEqual(os.WIFEXITED(status), True)
-self.assertEqual(os.WEXITSTATUS(status), os.EX_OK)
+proc = multiprocessing.Process(target=self._lock_subprocess,
args=(path,))
+proc.start()
+self.assertEqual(proc.pid > 0, True)
+proc.join()
+self.assertEqual(proc.exitcode, os.EX_OK)
portage.locks.unlockfile(lock1)
finally:
shutil.rmtree(tempdir)
+@staticmethod
+def _lock_subprocess(path):
+portage.locks._close_fds()
+# Disable close_fds since we don't exec
+# (see _setup_pipes docstring).
+portage.process._setup_pipes({0: 0, 1: 1, 2: 2}, close_fds=False)
+rval = 2
+try:
+try:
+lock2 = portage.locks.lockfile(path, flags=os.O_NONBLOCK)
+except portage.exception.TryAgain:
+rval = os.EX_OK
+else:
+rval = 1
+portage.locks.unlockfile(lock2)
+except Exception:
+traceback.print_exc()
+sys.exit(rval)
+
def testLockNonblock(self):
self._testLockNonblock()
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 3e10368e52ecead86b75478ca448ef0f59333e3e
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 4 04:34:45 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 7 00:55:46 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3e10368e
process.spawn: Use _start_proc for returnpid=False
This essentially completes the implementation of bug 916566,
eliminating os.fork() usage when "spawn" becomes the default
multiprocessing start method.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/portage/process.py | 35 +--
1 file changed, 17 insertions(+), 18 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 1ca59ee594..a33e7b4747 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -586,10 +586,10 @@ def spawn(
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
-mypids.extend(
+mypids.append(
spawn(
("tee", "-i", "-a", logfile),
-returnpid=True,
+returnproc=True,
fd_pipes={0: pr, 1: fd_pipes[1], 2: fd_pipes[2]},
)
)
@@ -634,7 +634,7 @@ def spawn(
# fork, so that the result is cached in the main process.
bool(groups)
-start_func = _start_proc if returnproc else _start_fork
+start_func = _start_proc if returnproc or not returnpid else _start_fork
pid = start_func(
_exec_wrapper,
@@ -666,7 +666,7 @@ def spawn(
# _start_proc returns a MultiprocessingProcess instance.
return pid
-if not isinstance(pid, int):
+if returnpid and not isinstance(pid, int):
raise AssertionError(f"fork returned non-integer: {repr(pid)}")
# Add the pid to our local and the global pid lists.
@@ -687,6 +687,8 @@ def spawn(
)
return mypids
+loop = global_event_loop()
+
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
@@ -695,25 +697,22 @@ def spawn(
pid = mypids.pop(0)
# and wait for it.
-retval = os.waitpid(pid, 0)[1]
+retval = loop.run_until_complete(pid.wait())
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
-# With waitpid and WNOHANG, only check the
-# first element of the tuple since the second
-# element may vary (bug #337465).
-if os.waitpid(pid, os.WNOHANG)[0] == 0:
-os.kill(pid, signal.SIGTERM)
-os.waitpid(pid, 0)
-
-# If it got a signal, return the signal that was sent.
-if retval & 0xFF:
-return (retval & 0xFF) << 8
-
-# Otherwise, return its exit code.
-return retval >> 8
+waiter = asyncio.ensure_future(pid.wait(), loop)
+try:
+loop.run_until_complete(
+asyncio.wait_for(asyncio.shield(waiter), 0.001)
+)
+except (TimeoutError, asyncio.TimeoutError):
+pid.terminate()
+loop.run_until_complete(waiter)
+
+return retval
# Everything succeeded
return 0
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: fa8e8f1895ed889aece2f67725df55d6ccf127fb Author: Zac Medico gentoo org> AuthorDate: Sat Feb 3 23:41:45 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 7 00:55:46 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa8e8f18 socks5: Migrate to spawn returnproc parameter Bug: https://bugs.gentoo.org/916566 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/socks5.py | 36 +++- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index fedb8599d5..6c68ff4106 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -1,10 +1,9 @@ # SOCKSv5 proxy manager for network-sandbox -# Copyright 2015-2021 Gentoo Authors +# Copyright 2015-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno import os -import signal import socket import portage.data @@ -22,7 +21,8 @@ class ProxyManager: def __init__(self): self.socket_path = None -self._pids = [] +self._proc = None +self._proc_waiter = None def start(self, settings): """ @@ -51,9 +51,9 @@ class ProxyManager: spawn_kwargs.update( uid=portage_uid, gid=portage_gid, groups=userpriv_groups, umask=0o077 ) -self._pids = spawn( +self._proc = spawn( [_python_interpreter, server_bin, self.socket_path], -returnpid=True, +returnproc=True, **spawn_kwargs, ) @@ -61,12 +61,19 @@ class ProxyManager: """ Stop the SOCKSv5 server. """ -for p in self._pids: -os.kill(p, signal.SIGINT) -os.waitpid(p, 0) +if self._proc is not None: +self._proc.terminate() +loop = asyncio.get_event_loop() +if self._proc_waiter is None: +self._proc_waiter = asyncio.ensure_future(self._proc.wait(), loop) +if loop.is_running(): +self._proc_waiter.add_done_callback(lambda future: future.result()) +else: +loop.run_until_complete(self._proc_waiter) self.socket_path = None -self._pids = [] +self._proc = None +self._proc_waiter = None def is_running(self): """ @@ -80,16 +87,11 @@ class ProxyManager: """ Wait for the proxy socket to become ready. This method is a coroutine. """ +if self._proc_waiter is None: +self._proc_waiter = asyncio.ensure_future(self._proc.wait()) while True: -try: -wait_retval = os.waitpid(self._pids[0], os.WNOHANG) -except OSError as e: -if e.errno == errno.EINTR: -continue -raise - -if wait_retval is not None and wait_retval != (0, 0): +if self._proc_waiter.done(): raise OSError(3, "No such process") try:
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 1ae4f574f69eca3146137ffb40c5afd8a4872777 Author: Zac Medico gentoo org> AuthorDate: Sun Feb 4 00:22:08 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 7 00:55:46 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1ae4f574 process.spawn: Enable returnpid warning for internals All internal returnpid consumers have been migrated to use the new returnproc parameter instead. Bug: https://bugs.gentoo.org/916566 Signed-off-by: Zac Medico gentoo.org> lib/portage/process.py | 11 +-- 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/portage/process.py b/lib/portage/process.py index 20327b38bc..1ca59ee594 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -680,12 +680,11 @@ def spawn( # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: -if not portage._internal_caller: -warnings.warn( -"The portage.process.spawn returnpid paramenter is deprecated and replaced by returnproc", -UserWarning, -stacklevel=1, -) +warnings.warn( +"The portage.process.spawn returnpid parameter is deprecated and replaced by returnproc", +UserWarning, +stacklevel=1, +) return mypids # Otherwise we clean them up.
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/sync/modules/rsync/
commit: 62332ee82b8b88fa5a65aafa7c221ccdaa7d65a8
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 4 00:11:07 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 7 00:55:46 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=62332ee8
RsyncSync: Migrate to spawn returnproc parameter
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/portage/sync/modules/rsync/rsync.py | 40 +--
lib/portage/util/futures/_asyncio/__init__.py | 6 +++-
2 files changed, 24 insertions(+), 22 deletions(-)
diff --git a/lib/portage/sync/modules/rsync/rsync.py
b/lib/portage/sync/modules/rsync/rsync.py
index 175c7f2e8e..5d442d2626 100644
--- a/lib/portage/sync/modules/rsync/rsync.py
+++ b/lib/portage/sync/modules/rsync/rsync.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import datetime
@@ -708,48 +708,47 @@ class RsyncSync(NewBase):
command.append(syncuri.rstrip("/") + "/metadata/timestamp.chk")
command.append(tmpservertimestampfile)
content = None
-pids = []
+proc = None
+proc_waiter = None
+loop = asyncio.get_event_loop()
try:
# Timeout here in case the server is unresponsive. The
# --timeout rsync option doesn't apply to the initial
# connection attempt.
try:
-if self.rsync_initial_timeout:
-
portage.exception.AlarmSignal.register(self.rsync_initial_timeout)
-
-pids.extend(
-portage.process.spawn(command, returnpid=True,
**self.spawn_kwargs)
+proc = portage.process.spawn(
+command, returnproc=True, **self.spawn_kwargs
+)
+proc_waiter = asyncio.ensure_future(proc.wait(), loop)
+future = (
+asyncio.wait_for(
+asyncio.shield(proc_waiter), self.rsync_initial_timeout
+)
+if self.rsync_initial_timeout
+else proc_waiter
)
-exitcode = os.waitpid(pids[0], 0)[1]
+exitcode = loop.run_until_complete(future)
if self.usersync_uid is not None:
portage.util.apply_permissions(
tmpservertimestampfile, uid=os.getuid()
)
content = portage.grabfile(tmpservertimestampfile)
finally:
-if self.rsync_initial_timeout:
-portage.exception.AlarmSignal.unregister()
try:
os.unlink(tmpservertimestampfile)
except OSError:
pass
-except portage.exception.AlarmSignal:
+except (TimeoutError, asyncio.TimeoutError):
# timed out
print("timed out")
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
-if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
-os.kill(pids[0], signal.SIGTERM)
-os.waitpid(pids[0], 0)
+if proc_waiter and not proc_waiter.done():
+proc.terminate()
+loop.run_until_complete(proc_waiter)
# This is the same code rsync uses for timeout.
exitcode = 30
-else:
-if exitcode != os.EX_OK:
-if exitcode & 0xFF:
-exitcode = (exitcode & 0xFF) << 8
-else:
-exitcode = exitcode >> 8
if content:
try:
@@ -758,7 +757,6 @@ class RsyncSync(NewBase):
)
except (OverflowError, ValueError):
pass
-del command, pids, content
if exitcode == os.EX_OK:
if (servertimestamp != 0) and (servertimestamp == timestamp):
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index a5a6cb3a5b..8f1b8e8275 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018-2021 Gentoo Authors
+# Copyright 2018-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = (
@@ -15,9 +15,11 @@ __all__ = (
"set_child_watcher",
"get_event_loop_policy",
"set_event_loop_policy",
+"shield",
"sleep",
"Task",
"wait",
+"wait_for",
)
import types
@@ -33,7 +35,9 @@ from asyncio import (
FIRST_EXCEPTION,
Future,
InvalidStateError,
+shield,
TimeoutError,
+wait_for,
)
import threading
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/executor/, lib/portage/tests/util/futures/
commit: 6c4b542b2a830587869b6180e879b719e97fda66 Author: Zac Medico gentoo org> AuthorDate: Tue Feb 6 04:00:29 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 7 00:49:26 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c4b542b ForkExecutor: multiprocessing spawn compat Use the AsyncFunction create_pipe=False parameter to avoid issues in the pipe code triggered with the "spawn" multiprocessing start method when spawn uses multiprocessing.Process (bug 916566), since these jobs should inherit stdio streams and run in the foreground with no log. Also fix RetryForkExecutorTestCase to avoid pickling issues. Bug: https://bugs.gentoo.org/923854 Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/futures/test_retry.py | 42 ++-- lib/portage/util/futures/executor/fork.py| 6 ++-- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/lib/portage/tests/util/futures/test_retry.py b/lib/portage/tests/util/futures/test_retry.py index a5b56bdc7f..6bd3f4b64a 100644 --- a/lib/portage/tests/util/futures/test_retry.py +++ b/lib/portage/tests/util/futures/test_retry.py @@ -1,4 +1,4 @@ -# Copyright 2018-2023 Gentoo Authors +# Copyright 2018-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from concurrent.futures import Future, ThreadPoolExecutor @@ -9,7 +9,6 @@ import threading import weakref import time -import portage from portage.tests import TestCase from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.backoff import RandomExponentialBackoff @@ -229,16 +228,19 @@ class RetryForkExecutorTestCase(RetryTestCase): @contextlib.contextmanager def _wrap_coroutine_func(self, coroutine_func): +uses_subprocess = isinstance(self._executor, ForkExecutor) parent_loop = global_event_loop() -parent_pid = portage.getpid() pending = weakref.WeakValueDictionary() # Since ThreadPoolExecutor does not propagate cancellation of a # parent_future to the underlying coroutine, use kill_switch to # propagate task cancellation to wrapper, so that HangForever's # thread returns when retry eventually cancels parent_future. -def wrapper(kill_switch): -if portage.getpid() == parent_pid: +if uses_subprocess: +wrapper = _run_coroutine_in_subprocess(coroutine_func) +else: + +def wrapper(kill_switch): # thread in main process def done_callback(result): result.cancelled() or result.exception() or result.result() @@ -262,22 +264,19 @@ class RetryForkExecutorTestCase(RetryTestCase): else: return future.result().result() -# child process -loop = global_event_loop() -try: -return loop.run_until_complete(coroutine_func()) -finally: -loop.close() - def execute_wrapper(): -kill_switch = threading.Event() +# Use kill_switch for threads because they can't be killed +# like processes. Do not pass kill_switch to subprocesses +# because it is not picklable. +kill_switch = None if uses_subprocess else threading.Event() +wrapper_args = [kill_switch] if kill_switch else [] parent_future = asyncio.ensure_future( -parent_loop.run_in_executor(self._executor, wrapper, kill_switch), +parent_loop.run_in_executor(self._executor, wrapper, *wrapper_args), loop=parent_loop, ) def kill_callback(parent_future): -if not kill_switch.is_set(): +if kill_switch is not None and not kill_switch.is_set(): kill_switch.set() parent_future.add_done_callback(kill_callback) @@ -298,6 +297,19 @@ class RetryForkExecutorTestCase(RetryTestCase): future.cancelled() or future.exception() or future.result() +class _run_coroutine_in_subprocess: +def __init__(self, coroutine_func): +self._coroutine_func = coroutine_func + +def __call__(self): +# child process +loop = global_event_loop() +try: +return loop.run_until_complete(self._coroutine_func()) +finally: +loop.close() + + class RetryThreadExecutorTestCase(RetryForkExecutorTestCase): def _setUpExecutor(self): self._executor = ThreadPoolExecutor(max_workers=1) diff --git a/lib/portage/util/futures/executor/fork.py b/lib/portage/util/futures/executor/fork.py index 61ad6aecfb..1e3d010724 100644 --- a/lib/portage/util/futures/executor/fork.py +++ b/lib/portage/util/futures/executor/fork.py @@ -1,4 +1,4 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2024 Gentoo Authors # Distributed under the
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 7d7ef237f3dddcf450fedab5aabfd57d1fb3406d Author: Zac Medico gentoo org> AuthorDate: Tue Feb 6 01:35:25 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 7 00:36:56 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7d7ef237 ForkProcess: Handle BrokenPipeError in _send_fd_pipes Convert _send_fd_pipes BrokenPipeError to asyncio.CancelledError, in order to gracefully handle a concurrently terminated child process as in testAsynchronousLockWaitCancel. Even if the child terminated abnormally, then there is no harm in suppressing the exception here, since the child error should have gone to stderr. Bug: https://bugs.gentoo.org/923852 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_async/ForkProcess.py | 25 + 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/lib/portage/util/_async/ForkProcess.py b/lib/portage/util/_async/ForkProcess.py index cb240d0712..ebcbd94107 100644 --- a/lib/portage/util/_async/ForkProcess.py +++ b/lib/portage/util/_async/ForkProcess.py @@ -153,15 +153,24 @@ class ForkProcess(SpawnProcess): This performs blocking IO, intended for invocation via run_in_executor. """ fd_list = list(set(self._fd_pipes.values())) -self._files.connection.send( -(self._fd_pipes, fd_list), -) -for fd in fd_list: -multiprocessing.reduction.send_handle( -self._files.connection, -fd, -self.pid, +try: +self._files.connection.send( +(self._fd_pipes, fd_list), ) +for fd in fd_list: +multiprocessing.reduction.send_handle( +self._files.connection, +fd, +self.pid, +) +except BrokenPipeError as e: +# This case is triggered by testAsynchronousLockWaitCancel +# when the test case terminates the child process while +# this thread is still sending the fd_pipes (bug 923852). +# Even if the child terminated abnormally, then there is +# no harm in suppressing the exception here, since the +# child error should have gone to stderr. +raise asyncio.CancelledError from e # self._fd_pipes contains duplicates that must be closed. for fd in fd_list:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/asyncio/
commit: 66d8f8388e5b9da7e07510f78ec487913e2ceaf5
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 4 00:16:29 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 7 00:36:17 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=66d8f838
ChildWatcherTestCase: Remove obsolete test which uses spawn returnpid
This test was added for bug 649588 when there was still an
internal event loop implementation for python2. It is no
longer relevant and uses the deprecated spawn returnpid
parameter, so remove it.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/futures/asyncio/meson.build | 1 -
.../util/futures/asyncio/test_child_watcher.py | 50 --
2 files changed, 51 deletions(-)
diff --git a/lib/portage/tests/util/futures/asyncio/meson.build
b/lib/portage/tests/util/futures/asyncio/meson.build
index ba727052fc..2de0668d6b 100644
--- a/lib/portage/tests/util/futures/asyncio/meson.build
+++ b/lib/portage/tests/util/futures/asyncio/meson.build
@@ -1,6 +1,5 @@
py.install_sources(
[
-'test_child_watcher.py',
'test_event_loop_in_fork.py',
'test_pipe_closed.py',
'test_policy_wrapper_recursion.py',
diff --git a/lib/portage/tests/util/futures/asyncio/test_child_watcher.py
b/lib/portage/tests/util/futures/asyncio/test_child_watcher.py
deleted file mode 100644
index cd100598b7..00
--- a/lib/portage/tests/util/futures/asyncio/test_child_watcher.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Copyright 2018-2021 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-import os
-
-from portage.process import find_binary, spawn
-from portage.tests import TestCase
-from portage.util._eventloop.global_event_loop import global_event_loop
-from portage.util.futures import asyncio
-from portage.util.futures.unix_events import DefaultEventLoopPolicy
-
-
-class ChildWatcherTestCase(TestCase):
-def testChildWatcher(self):
-true_binary = find_binary("true")
-self.assertNotEqual(true_binary, None)
-
-initial_policy = asyncio.get_event_loop_policy()
-if not isinstance(initial_policy, DefaultEventLoopPolicy):
-asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
-
-loop = None
-try:
-try:
-asyncio.set_child_watcher(None)
-except NotImplementedError:
-pass
-else:
-self.assertTrue(False)
-
-args_tuple = ("hello", "world")
-
-loop = asyncio._wrap_loop()
-future = loop.create_future()
-
-def callback(pid, returncode, *args):
-future.set_result((pid, returncode, args))
-
-async def watch_pid():
-with asyncio.get_child_watcher() as watcher:
-pids = spawn([true_binary], returnpid=True)
-watcher.add_child_handler(pids[0], callback, *args_tuple)
-self.assertEqual((await future), (pids[0], os.EX_OK,
args_tuple))
-
-loop.run_until_complete(watch_pid())
-finally:
-asyncio.set_event_loop_policy(initial_policy)
-if loop not in (None, global_event_loop()):
-loop.close()
-self.assertFalse(global_event_loop().is_closed())
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/bin/
commit: a5d92fafab93f62e5b28b97a443e0e06a368251d
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 5 22:51:47 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 5 22:54:04 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a5d92faf
tests/bin/setup_env.py: multiprocessing spawn compat
Replace unpicklable local pre_exec function with spawn
cwd argument.
Bug: https://bugs.gentoo.org/914876
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/bin/setup_env.py | 7 ++-
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/lib/portage/tests/bin/setup_env.py
b/lib/portage/tests/bin/setup_env.py
index faef118b05..5787f87682 100644
--- a/lib/portage/tests/bin/setup_env.py
+++ b/lib/portage/tests/bin/setup_env.py
@@ -1,5 +1,5 @@
# setup_env.py -- Make sure bin subdir has sane env for testing
-# Copyright 2007-2013 Gentoo Foundation
+# Copyright 2007-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import tempfile
@@ -78,10 +78,7 @@ def portage_func(func, args, exit_status=0):
f = open("/dev/null", "wb")
fd_pipes = {0: 0, 1: f.fileno(), 2: f.fileno()}
-def pre_exec():
-os.chdir(env["S"])
-
-spawn([func] + args.split(), env=env, fd_pipes=fd_pipes, pre_exec=pre_exec)
+spawn([func] + args.split(), env=env, fd_pipes=fd_pipes, cwd=env["S"])
f.close()
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: f97e414ce980299acc962e357db24106d62e4c7c
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 4 06:12:00 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 4 08:27:37 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f97e414c
set_term_size: Wait asynchronously if event loop is running
When set_term_size is called from an asynchronous context, we
can wait for the process to exit asynchronously. This will
prevent a "RuntimeError: This event loop is already running"
error in the future when synchronous spawn calls will need
to run the event loop in order to wait for the spawned
process(es).
Bug: https://bugs.gentoo.org/923750
Signed-off-by: Zac Medico gentoo.org>
lib/portage/output.py | 15 ---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/lib/portage/output.py b/lib/portage/output.py
index cdeeb18e99..7d3a6278f3 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2021 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__docformat__ = "epytext"
@@ -13,7 +13,9 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"portage.process:spawn",
"portage.util:writemsg",
+"portage.util.futures:asyncio",
)
import portage.util.formatter as formatter
@@ -557,13 +559,20 @@ def set_term_size(lines, columns, fd):
Set the number of lines and columns for the tty that is connected to fd.
For portability, this simply calls `stty rows $lines columns $columns`.
"""
-from portage.process import spawn
cmd = ["stty", "rows", str(lines), "columns", str(columns)]
try:
-spawn(cmd, env=os.environ, fd_pipes={0: fd})
+proc = spawn(cmd, env=os.environ, fd_pipes={0: fd}, returnproc=True)
except CommandNotFound:
writemsg(_("portage: stty: command not found\n"), noiselevel=-1)
+else:
+loop = asyncio.get_event_loop()
+if loop.is_running():
+asyncio.ensure_future(proc.wait(), loop).add_done_callback(
+lambda future: future.result()
+)
+else:
+loop.run_until_complete(proc.wait())
class EOutput:
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: c65d2d8b6c17d849e85f8c13c1a287ff1a07ccbd Author: Zac Medico gentoo org> AuthorDate: Sun Feb 4 07:22:37 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Feb 4 08:27:37 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c65d2d8b process.spawn: Avoid os.environ pickling error This error was observed when pickling os.environ for muliprocessing start method "spawn" after set_term_size was updated to use returnproc: File "/usr/lib/python3.12/multiprocessing/reduction.py", line 60, in dump ForkingPickler(file, protocol).dump(obj) AttributeError: Can't pickle local object '_createenviron..encode'. Did you mean: '_download_dir'? Bug: https://bugs.gentoo.org/923750 Signed-off-by: Zac Medico gentoo.org> lib/portage/process.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/portage/process.py b/lib/portage/process.py index 01426179d7..b223ecb887 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -527,6 +527,9 @@ def spawn( mycommand = mycommand.split() env = os.environ if env is None else env +# Sometimes os.environ can fail to pickle as shown in bug 923750 +# comment 4, so copy it to a dict. +env = env if isinstance(env, dict) else dict(env) env_stats = None if warn_on_large_env:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/, lib/_emerge/
commit: 052a4076cdf29fde8481646c636190d9db35f0ff
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 4 08:08:01 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 4 08:10:12 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=052a4076
Revert "process.spawn: Use multiprocessing.Process for returnproc"
This reverts commit 305612d1b04aa06d3d1a1c8b51d046a644742fd5.
It triggered a "Bad file descriptor" during the instprep
phase as reported in bug 923755.
Bug: https://bugs.gentoo.org/923755
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/SpawnProcess.py| 4 +-
lib/portage/process.py | 72 --
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 6 +-
3 files changed, 15 insertions(+), 67 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 5e582e3322..7f4a23892b 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -224,9 +224,7 @@ class SpawnProcess(SubProcess):
got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
-def _spawn(
-self, args: list[str], **kwargs
-) -> portage.process.MultiprocessingProcess:
+def _spawn(self, args: list[str], **kwargs) -> portage.process.Process:
spawn_func = portage.process.spawn
if self._selinux_type is not None:
diff --git a/lib/portage/process.py b/lib/portage/process.py
index d64ffa924f..01426179d7 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -19,7 +19,7 @@ import warnings
from dataclasses import dataclass
from functools import lru_cache
-from typing import Any, Optional, Callable, Union
+from typing import Any, Optional, Callable
from portage import os
from portage import _encodings
@@ -28,7 +28,6 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
-"portage.util._async.ForkProcess:ForkProcess",
"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util.futures:asyncio",
"portage.util:dump_traceback,writemsg,writemsg_level",
@@ -297,19 +296,12 @@ class AbstractProcess:
class Process(AbstractProcess):
"""
-An object that wraps OS processes which do not have an
-associated multiprocessing.Process instance. Ultimately,
-we need to stop using os.fork() to create these processes
-because it is unsafe for threaded processes as discussed
-in https://github.com/python/cpython/issues/84559.
-
-Note that if subprocess.Popen is used without pass_fds
-or preexec_fn parameters, then it avoids using os.fork()
-by instead using posix_spawn. This approach is not used
-by spawn because it needs to execute python code prior
-to exec, so it instead uses multiprocessing.Process,
-which only uses os.fork() when the multiprocessing start
-method is fork.
+An object that wraps OS processes created by spawn.
+In the future, spawn will return objects of a different type
+but with a compatible interface to this class, in order
+to encapsulate implementation-dependent objects like
+multiprocessing.Process which are designed to manage
+the process lifecycle and need to persist until it exits.
"""
def __init__(self, pid: int):
@@ -469,7 +461,7 @@ def spawn(
unshare_mount=False,
unshare_pid=False,
warn_on_large_env=False,
-) -> Union[int, MultiprocessingProcess, list[int]]:
+):
"""
Spawns a given command.
@@ -487,8 +479,8 @@ def spawn(
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will
clean them.
@type returnpid: Boolean
-@param returnproc: Return a MultiprocessingProcess instance (conflicts
with logfile parameter).
-NOTE: This requires the caller to asynchronously wait for the
MultiprocessingProcess instance.
+@param returnproc: Return a Process object for a successful spawn
(conflicts with logfile parameter).
+NOTE: This requires the caller to asynchronously wait for the Process.
@type returnproc: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@@ -631,9 +623,7 @@ def spawn(
# fork, so that the result is cached in the main process.
bool(groups)
-start_func = _start_proc if returnproc else _start_fork
-
-pid = start_func(
+pid = _start_fork(
_exec_wrapper,
args=(
binary,
@@ -659,10 +649,6 @@ def spawn(
close_fds=close_fds,
)
-if returnproc:
-# _start_proc returns a MultiprocessingProcess instance.
-return pid
-
if not isinstance(pid, int):
raise AssertionError(f"fork returned non-integer: {repr(pid)}")
@@ -684,6 +670,8 @@ def spawn(
stacklevel=1,
)
return mypids
+
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 9f96ce5105e7bd2580ae9acc34d6ebad914dae47
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 3 19:36:05 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 3 19:55:19 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9f96ce51
ForkProcess: Use duplicate fd_pipes in _send_fd_pipes thread
In order to allow callers to manage the lifecycle of fd_pipes
file descriptors, create duplicates for _send_fd_pipes to
close when it has finished sending them.
This fixes bug 916601 in a nice way, allowing commit
3b1234ba69a31709cd5aec1ae070901e3a28bb7c to be reverted.
Bug: https://bugs.gentoo.org/916601
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_async/ForkProcess.py | 55 +-
1 file changed, 48 insertions(+), 7 deletions(-)
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index 711bd2a7ba..3acbe34fc6 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -24,6 +24,8 @@ class ForkProcess(SpawnProcess):
"kwargs",
"target",
"_child_connection",
+# Duplicate file descriptors for use by _send_fd_pipes background
thread.
+"_fd_pipes",
)
_file_names = ("connection", "slave_fd")
@@ -53,7 +55,13 @@ class ForkProcess(SpawnProcess):
duplex=self._HAVE_SEND_HANDLE
)
-self._proc = self._spawn(self.args, fd_pipes=self.fd_pipes)
+# Handle fd_pipes in _main instead, since file descriptors are
+# not inherited with the multiprocessing "spawn" start method.
+# Pass fd_pipes=None to spawn here so that it doesn't leave
+# a closed stdin duplicate in fd_pipes (that would trigger
+# "Bad file descriptor" error if we tried to send it via
+# send_handle).
+self._proc = self._spawn(self.args, fd_pipes=None)
self._registered = True
@@ -74,6 +82,25 @@ class ForkProcess(SpawnProcess):
self.fd_pipes[1] = slave_fd
self.fd_pipes[2] = slave_fd
self._files = self._files_dict(connection=connection,
slave_fd=slave_fd)
+
+# Create duplicate file descriptors in self._fd_pipes
+# so that the caller is free to manage the lifecycle
+# of the original fd_pipes.
+self._fd_pipes = {}
+fd_map = {}
+for dest, src in list(self.fd_pipes.items()):
+if src not in fd_map:
+src_new = fd_map[src] = os.dup(src)
+old_fdflags = fcntl.fcntl(src, fcntl.F_GETFD)
+fcntl.fcntl(src_new, fcntl.F_SETFD, old_fdflags)
+os.set_inheritable(
+src_new, not bool(old_fdflags & fcntl.FD_CLOEXEC)
+)
+self._fd_pipes[dest] = fd_map[src]
+
+asyncio.ensure_future(
+self._proc.wait(), self.scheduler
+).add_done_callback(self._close_fd_pipes)
else:
master_fd = connection
@@ -81,6 +108,19 @@ class ForkProcess(SpawnProcess):
master_fd, log_file_path=self.logfile, stdout_fd=stdout_fd
)
+def _close_fd_pipes(self, future):
+"""
+Cleanup self._fd_pipes if needed, since _send_fd_pipes could
+have been cancelled.
+"""
+# future.result() raises asyncio.CancelledError if
+# future.cancelled(), but that should not happen.
+future.result()
+if self._fd_pipes is not None:
+for fd in set(self._fd_pipes.values()):
+os.close(fd)
+self._fd_pipes = None
+
@property
def _fd_pipes_send_handle(self):
"""Returns True if we have a connection to implement fd_pipes via
send_handle."""
@@ -95,9 +135,9 @@ class ForkProcess(SpawnProcess):
Communicate with _bootstrap to send fd_pipes via send_handle.
This performs blocking IO, intended for invocation via run_in_executor.
"""
-fd_list = list(set(self.fd_pipes.values()))
+fd_list = list(set(self._fd_pipes.values()))
self._files.connection.send(
-(self.fd_pipes, fd_list),
+(self._fd_pipes, fd_list),
)
for fd in fd_list:
multiprocessing.reduction.send_handle(
@@ -106,6 +146,11 @@ class ForkProcess(SpawnProcess):
self.pid,
)
+# self._fd_pipes contains duplicates that must be closed.
+for fd in fd_list:
+os.close(fd)
+self._fd_pipes = None
+
async def _main(self, build_logger, pipe_logger, loop=None):
try:
if self._fd_pipes_send_handle:
@@ -167,10 +212,6 @@ class ForkProcess(SpawnProcess):
)
fd_pipes[0] = stdin_dup
-
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/
commit: 7ea1175091886baa677d11290d4b725a64e68710
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 3 19:36:13 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 3 19:55:19 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7ea11750
Revert "testAsyncFunctionStdin: multiprocessing spawn compat"
This reverts commit 3b1234ba69a31709cd5aec1ae070901e3a28bb7c,
since ForkProcess now solves the problem by creating temporary
duplicate file descriptors.
Bug: https://bugs.gentoo.org/916601
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/process/test_AsyncFunction.py | 19 ---
1 file changed, 8 insertions(+), 11 deletions(-)
diff --git a/lib/portage/tests/process/test_AsyncFunction.py
b/lib/portage/tests/process/test_AsyncFunction.py
index a056f268bd..eb426a5c02 100644
--- a/lib/portage/tests/process/test_AsyncFunction.py
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Gentoo Authors
+# Copyright 2020-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -44,20 +44,17 @@ class AsyncFunctionTestCase(TestCase):
),
)
reader.start()
-# For compatibility with the multiprocessing spawn start
-# method, we delay restoration of the stdin file descriptor,
-# since this file descriptor is sent to the subprocess
-# asynchronously.
-_set_nonblocking(pw.fileno())
-with open(pw.fileno(), mode="wb", buffering=0, closefd=False) as
pipe_write:
-await _writer(pipe_write, test_string.encode("utf_8"))
-pw.close()
-self.assertEqual((await reader.async_wait()), os.EX_OK)
-self.assertEqual(reader.result, test_string)
finally:
os.dup2(stdin_backup, portage._get_stdin().fileno())
os.close(stdin_backup)
+_set_nonblocking(pw.fileno())
+with open(pw.fileno(), mode="wb", buffering=0, closefd=False) as
pipe_write:
+await _writer(pipe_write, test_string.encode("utf_8"))
+pw.close()
+self.assertEqual((await reader.async_wait()), os.EX_OK)
+self.assertEqual(reader.result, test_string)
+
def testAsyncFunctionStdin(self):
loop = asyncio._wrap_loop()
loop.run_until_complete(self._testAsyncFunctionStdin(loop=loop))
[gentoo-commits] proj/portage:master commit in: lib/portage/, lib/portage/package/ebuild/, lib/_emerge/, ...
commit: 055c66ec9482064aaaf51bfb6b01e260ea27808e
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 2 16:01:18 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 2 16:01:18 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=055c66ec
SpawnProcess: Use spawn returnproc parameter
Migrate SpawnProcess to use the spawn returnproc parameter,
and make adaptations to descendent classes as needed. Introduce a
portage.process.MultiprocessingProcess class for ForkProcess to wrap
multiprocessing.Process instances, needed because ForkProcess
inherits from SpawnProcess. Use portage.process.Process to wrap
the pid in EbuildMetadataPhase, so that returnproc support in
the doebuild function can be reserved for a later commit.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py | 4 +-
lib/_emerge/SpawnProcess.py | 16 ++---
lib/_emerge/SubProcess.py | 25 +++
lib/portage/package/ebuild/doebuild.py | 4 +-
lib/portage/process.py | 120
lib/portage/util/_async/ForkProcess.py | 87 +++
lib/portage/util/_async/PopenProcess.py | 5 +-
7 files changed, 143 insertions(+), 118 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index 8905a058fc..a7c9650d74 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.SubProcess import SubProcess
@@ -137,7 +137,7 @@ class EbuildMetadataPhase(SubProcess):
self._async_wait()
return
-self.pid = retval[0]
+self._proc = portage.process.Process(retval[0])
def _output_handler(self):
while True:
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 40740df9aa..7f4a23892b 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2008-2023 Gentoo Authors
+# Copyright 2008-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -123,24 +123,16 @@ class SpawnProcess(SubProcess):
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
-kwargs["returnpid"] = True
+kwargs["returnproc"] = True
kwargs.pop("logfile", None)
-retval = self._spawn(self.args, **kwargs)
+self._proc = self._spawn(self.args, **kwargs)
if slave_fd is not None:
os.close(slave_fd)
if null_input is not None:
os.close(null_input)
-if isinstance(retval, int):
-# spawn failed
-self.returncode = retval
-self._async_wait()
-return
-
-self.pid = retval[0]
-
if not fd_pipes:
self._registered = True
self._async_waitpid()
@@ -232,7 +224,7 @@ class SpawnProcess(SubProcess):
got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
-def _spawn(self, args, **kwargs):
+def _spawn(self, args: list[str], **kwargs) -> portage.process.Process:
spawn_func = portage.process.spawn
if self._selinux_type is not None:
diff --git a/lib/_emerge/SubProcess.py b/lib/_emerge/SubProcess.py
index b734591d11..029bbc3f44 100644
--- a/lib/_emerge/SubProcess.py
+++ b/lib/_emerge/SubProcess.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import logging
@@ -12,12 +12,16 @@ import errno
class SubProcess(AbstractPollTask):
-__slots__ = ("pid",) + ("_dummy_pipe_fd", "_files", "_waitpid_id")
+__slots__ = ("_dummy_pipe_fd", "_files", "_proc", "_waitpid_id")
# This is how much time we allow for waitpid to succeed after
# we've sent a kill signal to our subprocess.
_cancel_timeout = 1 # seconds
+@property
+def pid(self):
+return self._proc.pid
+
def _poll(self):
# Simply rely on _async_waitpid_cb to set the returncode.
return self.returncode
@@ -58,15 +62,11 @@ class SubProcess(AbstractPollTask):
if self.returncode is not None:
self._async_wait()
elif self._waitpid_id is None:
-self._waitpid_id = self.pid
-self.scheduler._asyncio_child_watcher.add_child_handler(
-self.pid, self._async_waitpid_cb
-)
-
-def _async_waitpid_cb(self, pid, returncode):
-if pid != self.pid:
-raise AssertionError(f"expected pid {self.pid}, got {pid}")
-self.returncode = returncode
+self._waitpid_id =
[gentoo-commits] proj/portage:master commit in: lib/portage/, lib/portage/tests/process/
commit: c556fee09b6b103a9bb8c2afa1c7a7ef25022598
Author: Zac Medico gentoo org>
AuthorDate: Thu Feb 1 05:48:54 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Feb 1 16:10:28 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c556fee0
process.spawn: Add returnproc parameter
In order to migrate away from unsafe os.fork() usage in threaded
processes (https://github.com/python/cpython/issues/84559), add a
returnproc parameter that is similar to returnpid, which causes
spawn to return a single Process object instead of a list of pids.
The Process API is a subset of asyncio.subprocess.Process. The
returnproc parameter conflicts with the logfile parameter, since
the caller is expected to use the fd_pipes parameter to implement
logging (this was also true for the returnpid parameter).
In the future, spawn will return objects of a different type but
with a compatible interface to Process, in order to encapsulate
implementation-dependent objects like multiprocessing.Process which
are designed to manage the process lifecycle and need to persist
until it exits.
Trigger a UserWarning when the returnpid parameter is used, in
order to encourage migration to returnproc (do not use
DeprecationWarning since it is hidden by default). This warning
will be temporarily suppressed for portage internals, until they
finish migrating to returnproc. There are probably very few if
any external consumers of spawn with the returnpid parameter,
so it seems safe to move quickly with this deprecation.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/portage/process.py | 84 ++
lib/portage/tests/process/meson.build | 1 +
lib/portage/tests/process/test_spawn_returnproc.py | 39 ++
3 files changed, 124 insertions(+)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 71750a715f..6ec52efc4a 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -15,6 +15,7 @@ import subprocess
import sys
import traceback
import os as _os
+import warnings
from dataclasses import dataclass
from functools import lru_cache
@@ -27,6 +28,7 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util:dump_traceback,writemsg,writemsg_level",
)
@@ -277,12 +279,78 @@ def calc_env_stats(env) -> EnvStats:
env_too_large_warnings = 0
+class Process:
+"""
+An object that wraps OS processes created by spawn.
+In the future, spawn will return objects of a different type
+but with a compatible interface to this class, in order
+to encapsulate implementation-dependent objects like
+multiprocessing.Process which are designed to manage
+the process lifecycle and need to persist until it exits.
+"""
+
+def __init__(self, pid):
+self.pid = pid
+self.returncode = None
+self._exit_waiters = []
+
+def __repr__(self):
+return f"<{self.__class__.__name__} {self.pid}>"
+
+async def wait(self):
+"""
+Wait for the child process to terminate.
+
+Set and return the returncode attribute.
+"""
+if self.returncode is not None:
+return self.returncode
+
+loop = global_event_loop()
+if not self._exit_waiters:
+loop._asyncio_child_watcher.add_child_handler(self.pid,
self._child_handler)
+waiter = loop.create_future()
+self._exit_waiters.append(waiter)
+return await waiter
+
+def _child_handler(self, pid, returncode):
+if pid != self.pid:
+raise AssertionError(f"expected pid {self.pid}, got {pid}")
+self.returncode = returncode
+
+for waiter in self._exit_waiters:
+if not waiter.cancelled():
+waiter.set_result(returncode)
+self._exit_waiters = None
+
+def send_signal(self, sig):
+"""Send a signal to the process."""
+if self.returncode is not None:
+# Skip signalling a process that we know has already died.
+return
+
+try:
+os.kill(self.pid, sig)
+except ProcessLookupError:
+# Suppress the race condition error; bpo-40550.
+pass
+
+def terminate(self):
+"""Terminate the process with SIGTERM"""
+self.send_signal(signal.SIGTERM)
+
+def kill(self):
+"""Kill the process with SIGKILL"""
+self.send_signal(signal.SIGKILL)
+
+
def spawn(
mycommand,
env=None,
opt_name=None,
fd_pipes=None,
returnpid=False,
+returnproc=False,
uid=None,
gid=None,
groups=None,
@@ -315,6 +383,9 @@ def spawn(
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will
clean them.
@type returnpid: Boolean
+
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: c82dc8c8fe92979b82df4c71bb9961973367e8f9
Author: Zac Medico gentoo org>
AuthorDate: Mon Jan 29 17:38:20 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jan 29 17:43:27 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c82dc8c8
_start_fork: Ensure any _setup_pipes exception is displayed
Fixes: 7ec7a647ef6e ("process.spawn: add abstraction layer for os.fork()")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/process.py | 17 +
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index be1c5d350a..71750a715f 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -658,13 +658,8 @@ def _exec_wrapper(
writemsg(
f"ERROR: Executing {mycommand} failed with E2BIG. Child
process environment size: {env_stats.env_size} bytes. Largest environment
variable: {env_stats.env_largest_name} ({env_stats.env_largest_size} bytes)\n"
)
-
-# We need to catch _any_ exception so that it doesn't
-# propagate out of this function and cause exiting
-# with anything other than os._exit()
writemsg(f"{e}:\n {' '.join(mycommand)}\n", noiselevel=-1)
-traceback.print_exc()
-sys.stderr.flush()
+raise
def _exec(
@@ -1174,8 +1169,14 @@ def _start_fork(
pid = os.fork()
if pid == 0:
-_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
-target(*args, **kwargs)
+try:
+_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
+target(*args, **kwargs)
+except Exception:
+# We need to catch _any_ exception and display it since the
child
+# process must unconditionally exit via os._exit() if exec
fails.
+traceback.print_exc()
+sys.stderr.flush()
finally:
# Don't used portage.getpid() here, in case there is a race
# with getpid cache invalidation via _ForkWatcher hook.
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 7ec7a647ef6e2d94e6f2b387d0a012e78cafbaff
Author: Zac Medico gentoo org>
AuthorDate: Mon Jan 29 08:20:55 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jan 29 08:43:52 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7ec7a647
process.spawn: add abstraction layer for os.fork()
Since os.fork() is unsafe in threaded processes, add a basic
abstraction layer that will ultimately allow support of other process
cloning implementations.
Usage of os.fork() is now wrapped in a _start_fork function which can
easily be replaced with an alternative implementation. This function
takes target, args, and kwargs arguments which are equivalent to
the corresponding multiprocessing.Process parameters. It also has
fd_pipes and close_fds parameters, since implementation of these is
dependent on the process cloning implementation.
The process cloning implementation does not need to be bothered with
the details of the _exec function, since spawn uses an _exec_wrapper
function as a target function which calls _exec and handles any
exceptions appropriately (special exception handling is required for
success of test_spawnE2big related to bug 830187).
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/portage/process.py | 203 +
1 file changed, 139 insertions(+), 64 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 0d58adecad..be1c5d350a 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -18,6 +18,7 @@ import os as _os
from dataclasses import dataclass
from functools import lru_cache
+from typing import Any, Optional, Callable
from portage import os
from portage import _encodings
@@ -450,68 +451,31 @@ def spawn(
# fork, so that the result is cached in the main process.
bool(groups)
-parent_pid = portage.getpid()
-pid = None
-try:
-pid = os.fork()
-
-if pid == 0:
-try:
-_exec(
-binary,
-mycommand,
-opt_name,
-fd_pipes,
-env,
-gid,
-groups,
-uid,
-umask,
-cwd,
-pre_exec,
-close_fds,
-unshare_net,
-unshare_ipc,
-unshare_mount,
-unshare_pid,
-unshare_flags,
-)
-except SystemExit:
-raise
-except Exception as e:
-if isinstance(e, OSError) and e.errno == errno.E2BIG:
-# If exec() failed with E2BIG, then this is
-# potentially because the environment variables
-# grew to large. The following will gather some
-# stats about the environment and print a
-# diagnostic message to help identifying the
-# culprit. See also
-# - https://bugs.gentoo.org/721088
-# - https://bugs.gentoo.org/830187
-if not env_stats:
-env_stats = calc_env_stats(env)
-
-writemsg(
-f"ERROR: Executing {mycommand} failed with E2BIG.
Child process environment size: {env_stats.env_size} bytes. Largest environment
variable: {env_stats.env_largest_name} ({env_stats.env_largest_size} bytes)\n"
-)
-
-# We need to catch _any_ exception so that it doesn't
-# propagate out of this function and cause exiting
-# with anything other than os._exit()
-writemsg(f"{e}:\n {' '.join(mycommand)}\n", noiselevel=-1)
-traceback.print_exc()
-sys.stderr.flush()
-
-finally:
-# Don't used portage.getpid() here, in case there is a race
-# with getpid cache invalidation via _ForkWatcher hook.
-if pid == 0 or (pid is None and _os.getpid() != parent_pid):
-# Call os._exit() from a finally block in order
-# to suppress any finally blocks from earlier
-# in the call stack (see bug #345289). This
-# finally block has to be setup before the fork
-# in order to avoid a race condition.
-os._exit(1)
+pid = _start_fork(
+_exec_wrapper,
+args=(
+binary,
+mycommand,
+opt_name,
+fd_pipes,
+env,
+gid,
+groups,
+uid,
+umask,
+cwd,
+pre_exec,
+close_fds,
+unshare_net,
+unshare_ipc,
+unshare_mount,
+unshare_pid,
+unshare_flags,
+env_stats,
+),
+
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/, lib/portage/tests/locks/
commit: d4c495c6bd543d07f3c5d6e72145e177999edaab Author: Zac Medico gentoo org> AuthorDate: Sun Jan 28 23:44:23 2024 + Commit: Zac Medico gentoo org> CommitDate: Mon Jan 29 04:08:42 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d4c495c6 Rely on os.register_at_fork for _ForkWatcher hook invocation Because os.register_at_fork is used instead of multiprocessing.util.register_after_fork since commit cc7c40199850acb3d36f0a6452987231d592c360, we can rely on the _ForkWatcher hook being automatically invoked after os.fork(). Signed-off-by: Zac Medico gentoo.org> lib/portage/process.py| 10 +++--- lib/portage/tests/locks/test_lock_nonblock.py | 3 +-- lib/portage/util/locale.py| 3 +-- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/portage/process.py b/lib/portage/process.py index 28f977a046..0d58adecad 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -1,5 +1,5 @@ # portage.py -- core Portage functionality -# Copyright 1998-2023 Gentoo Authors +# Copyright 1998-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 @@ -456,7 +456,6 @@ def spawn( pid = os.fork() if pid == 0: -portage._ForkWatcher.hook(portage._ForkWatcher) try: _exec( binary, @@ -504,8 +503,8 @@ def spawn( sys.stderr.flush() finally: -# Don't used portage.getpid() here, due to a race with the above -# portage._ForkWatcher cache update. +# Don't used portage.getpid() here, in case there is a race +# with getpid cache invalidation via _ForkWatcher hook. if pid == 0 or (pid is None and _os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier @@ -775,9 +774,6 @@ def _exec( if unshare_pid: main_child_pid = os.fork() if main_child_pid == 0: -# The portage.getpid() cache may need to be updated here, -# in case the pre_exec function invokes portage APIs. -portage._ForkWatcher.hook(portage._ForkWatcher) # pid namespace requires us to become init binary, myargs = ( portage._python_interpreter, diff --git a/lib/portage/tests/locks/test_lock_nonblock.py b/lib/portage/tests/locks/test_lock_nonblock.py index e3f9b4d023..9bb91b428e 100644 --- a/lib/portage/tests/locks/test_lock_nonblock.py +++ b/lib/portage/tests/locks/test_lock_nonblock.py @@ -1,4 +1,4 @@ -# Copyright 2011-2020 Gentoo Authors +# Copyright 2011-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import tempfile @@ -19,7 +19,6 @@ class LockNonblockTestCase(TestCase): lock1 = portage.locks.lockfile(path) pid = os.fork() if pid == 0: -portage._ForkWatcher.hook(portage._ForkWatcher) portage.locks._close_fds() # Disable close_fds since we don't exec # (see _setup_pipes docstring). diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py index 0d0c120157..a620dbd544 100644 --- a/lib/portage/util/locale.py +++ b/lib/portage/util/locale.py @@ -1,4 +1,4 @@ -# Copyright 2015-2020 Gentoo Authors +# Copyright 2015-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 """ @@ -118,7 +118,6 @@ def check_locale(silent=False, env=None): pid = os.fork() if pid == 0: -portage._ForkWatcher.hook(portage._ForkWatcher) try: if env is not None: try:
[gentoo-commits] proj/portage:master commit in: lib/portage/util/file_copy/, lib/_emerge/, lib/portage/util/_async/, ...
commit: 3d55e159c473075c7b2f87c92293b0df6fa57563
Author: Zac Medico gentoo org>
AuthorDate: Sun Jan 28 22:01:58 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Jan 28 23:09:26 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d55e159
*/*: rerun black w/ 24.1.0
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/AsynchronousTask.py | 6 +--
lib/_emerge/EbuildMetadataPhase.py | 11 ++---
lib/_emerge/MergeListItem.py| 1 -
lib/_emerge/PipeReader.py | 1 -
lib/_emerge/SpawnProcess.py | 1 -
lib/_emerge/depgraph.py | 59 ++---
lib/_emerge/resolver/slot_collision.py | 12 ++---
lib/portage/_emirrordist/DeletionIterator.py| 8 ++--
lib/portage/_emirrordist/FetchIterator.py | 8 ++--
lib/portage/_sets/libs.py | 1 -
lib/portage/dbapi/porttree.py | 1 -
lib/portage/dbapi/vartree.py| 28 +++-
lib/portage/dep/__init__.py | 1 -
lib/portage/package/ebuild/config.py| 38
lib/portage/package/ebuild/doebuild.py | 16 ---
lib/portage/proxy/objectproxy.py| 1 -
lib/portage/tests/ebuild/test_fetch.py | 10 ++---
lib/portage/tests/process/test_AsyncFunction.py | 8 ++--
lib/portage/util/_async/PipeLogger.py | 1 -
lib/portage/util/_async/TaskScheduler.py| 1 -
lib/portage/util/_dyn_libs/LinkageMapELF.py | 3 --
lib/portage/util/file_copy/__init__.py | 7 +--
lib/portage/util/futures/_sync_decorator.py | 8 ++--
23 files changed, 119 insertions(+), 112 deletions(-)
diff --git a/lib/_emerge/AsynchronousTask.py b/lib/_emerge/AsynchronousTask.py
index 4290eede36..4049ba5eb6 100644
--- a/lib/_emerge/AsynchronousTask.py
+++ b/lib/_emerge/AsynchronousTask.py
@@ -46,9 +46,9 @@ class AsynchronousTask(SlotObject):
)
self.addExitListener(exit_listener)
waiter.add_done_callback(
-lambda waiter: self.removeExitListener(exit_listener)
-if waiter.cancelled()
-else None
+lambda waiter: (
+self.removeExitListener(exit_listener) if waiter.cancelled()
else None
+)
)
if self.returncode is not None:
# If the returncode is not None, it means the exit event has
already
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index fd695e0253..8905a058fc 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -19,7 +19,6 @@ import fcntl
class EbuildMetadataPhase(SubProcess):
-
"""
Asynchronous interface for the ebuild "depend" phase which is
used to extract metadata from the ebuild.
@@ -200,12 +199,10 @@ class EbuildMetadataPhase(SubProcess):
# entries for unsupported EAPIs.
if self.eapi_supported:
if metadata.get("INHERITED", False):
-metadata[
-"_eclasses_"
-] = self.portdb.repositories.get_repo_for_location(
-self.repo_path
-).eclass_db.get_eclass_data(
-metadata["INHERITED"].split()
+metadata["_eclasses_"] = (
+self.portdb.repositories.get_repo_for_location(
+self.repo_path
+
).eclass_db.get_eclass_data(metadata["INHERITED"].split())
)
else:
metadata["_eclasses_"] = {}
diff --git a/lib/_emerge/MergeListItem.py b/lib/_emerge/MergeListItem.py
index efe485c2e0..ae894704a4 100644
--- a/lib/_emerge/MergeListItem.py
+++ b/lib/_emerge/MergeListItem.py
@@ -13,7 +13,6 @@ from _emerge.PackageUninstall import PackageUninstall
class MergeListItem(CompositeTask):
-
"""
TODO: For parallel scheduling, everything here needs asynchronous
execution support (start, poll, and wait methods).
diff --git a/lib/_emerge/PipeReader.py b/lib/_emerge/PipeReader.py
index 026346e0bb..76ab7f1882 100644
--- a/lib/_emerge/PipeReader.py
+++ b/lib/_emerge/PipeReader.py
@@ -8,7 +8,6 @@ from _emerge.AbstractPollTask import AbstractPollTask
class PipeReader(AbstractPollTask):
-
"""
Reads output from one or more files and saves it in memory,
for retrieval via the getvalue() method. This is driven by
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 72fa72c613..40740df9aa 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -16,7 +16,6 @@ from portage.util.futures import asyncio
class SpawnProcess(SubProcess):
-
"""
Constructor keyword args are passed into
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_dyn_libs/, lib/portage/tests/util/dyn_libs/
commit: 95d3e5e80ab9561db870858c2caf6e3bffbf47b0 Author: Zac Medico gentoo org> AuthorDate: Mon Jan 15 20:30:57 2024 + Commit: Zac Medico gentoo org> CommitDate: Sat Jan 20 05:18:12 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=95d3e5e8 installed_dynlibs: Resolve *.so symlinks Resolve *.so symlinks to check if they point to regular files inside the top directory. If a symlink points outside the top directory then try to follow the corresponding file inside the top directory if it exists, and otherwise stop following. Bug: https://bugs.gentoo.org/921170 Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/dyn_libs/meson.build| 1 + .../tests/util/dyn_libs/test_installed_dynlibs.py | 65 ++ lib/portage/util/_dyn_libs/dyn_libs.py | 43 +- 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/lib/portage/tests/util/dyn_libs/meson.build b/lib/portage/tests/util/dyn_libs/meson.build index ddb08f5b1a..8f2c919c13 100644 --- a/lib/portage/tests/util/dyn_libs/meson.build +++ b/lib/portage/tests/util/dyn_libs/meson.build @@ -1,5 +1,6 @@ py.install_sources( [ +'test_installed_dynlibs.py', 'test_soname_deps.py', '__init__.py', '__test__.py', diff --git a/lib/portage/tests/util/dyn_libs/test_installed_dynlibs.py b/lib/portage/tests/util/dyn_libs/test_installed_dynlibs.py new file mode 100644 index 00..421dcf6061 --- /dev/null +++ b/lib/portage/tests/util/dyn_libs/test_installed_dynlibs.py @@ -0,0 +1,65 @@ +# Copyright 2024 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import os +import tempfile + +from portage.const import BASH_BINARY +from portage.tests import TestCase +from portage.util import ensure_dirs +from portage.util._dyn_libs.dyn_libs import installed_dynlibs +from portage.util.file_copy import copyfile + + +class InstalledDynlibsTestCase(TestCase): +def testInstalledDynlibsRegular(self): +""" +Return True for *.so regular files. +""" +with tempfile.TemporaryDirectory() as directory: +bash_copy = os.path.join(directory, "lib", "libfoo.so") +ensure_dirs(os.path.dirname(bash_copy)) +copyfile(BASH_BINARY, bash_copy) +self.assertTrue(installed_dynlibs(directory)) + +def testInstalledDynlibsOnlySymlink(self): +""" +If a *.so symlink is installed but does not point to a regular +file inside the top directory, installed_dynlibs should return +False (bug 921170). +""" +with tempfile.TemporaryDirectory() as directory: +symlink_path = os.path.join(directory, "lib", "libfoo.so") +ensure_dirs(os.path.dirname(symlink_path)) +os.symlink(BASH_BINARY, symlink_path) +self.assertFalse(installed_dynlibs(directory)) + +def testInstalledDynlibsSymlink(self): +""" +Return True for a *.so symlink pointing to a regular file inside +the top directory. +""" +with tempfile.TemporaryDirectory() as directory: +bash_copy = os.path.join(directory, BASH_BINARY.lstrip(os.sep)) +ensure_dirs(os.path.dirname(bash_copy)) +copyfile(BASH_BINARY, bash_copy) +symlink_path = os.path.join(directory, "lib", "libfoo.so") +ensure_dirs(os.path.dirname(symlink_path)) +os.symlink(bash_copy, symlink_path) +self.assertTrue(installed_dynlibs(directory)) + +def testInstalledDynlibsAbsoluteSymlink(self): +""" +If a *.so symlink target is outside of the top directory, +traversal follows the corresponding file inside the top +directory if it exists, and otherwise stops following the +symlink. +""" +with tempfile.TemporaryDirectory() as directory: +bash_copy = os.path.join(directory, BASH_BINARY.lstrip(os.sep)) +ensure_dirs(os.path.dirname(bash_copy)) +copyfile(BASH_BINARY, bash_copy) +symlink_path = os.path.join(directory, "lib", "libfoo.so") +ensure_dirs(os.path.dirname(symlink_path)) +os.symlink(BASH_BINARY, symlink_path) +self.assertTrue(installed_dynlibs(directory)) diff --git a/lib/portage/util/_dyn_libs/dyn_libs.py b/lib/portage/util/_dyn_libs/dyn_libs.py index ee28e8839c..6f8a07d70d 100644 --- a/lib/portage/util/_dyn_libs/dyn_libs.py +++ b/lib/portage/util/_dyn_libs/dyn_libs.py @@ -1,14 +1,51 @@ -# Copyright 2021 Gentoo Authors +# Copyright 2021-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os +import stat + +import portage def installed_dynlibs(directory): -for _dirpath, _dirnames, filenames in os.walk(directory): +""" +This traverses installed *.so symlinks to check if they point to +regular files. If a symlink
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: adc0052b2c022851458f788868e6b194ed1cfe9f
Author: Sam James gentoo org>
AuthorDate: Tue Jan 16 07:51:47 2024 +
Commit: Sam James gentoo org>
CommitDate: Tue Jan 16 07:52:11 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=adc0052b
bintree: add missing newlines to signed binpkg update notice
Signed-off-by: Sam James gentoo.org>
lib/portage/dbapi/bintree.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index ee574f435f..f4251b47d6 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -306,7 +306,7 @@ class bindbapi(fakedbapi):
colorize(
"WARN",
f"Binpkg update ignored for signed package:
{binpkg_path}, "
-"the file will be removed.",
+"the file will be removed.\n",
)
)
self.bintree.remove(cpv)
@@ -734,7 +734,7 @@ class binarytree:
writemsg(
colorize(
"WARN",
-f"Binpkg update ignored for signed package:
{binpkg_path}",
+f"Binpkg update ignored for signed package:
{binpkg_path}\n",
)
)
continue
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 68f4ea8a90d8759a1aa859d9188017e21797bdd0 Author: Zac Medico gentoo org> AuthorDate: Tue Jan 16 01:08:36 2024 + Commit: Zac Medico gentoo org> CommitDate: Tue Jan 16 05:25:20 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=68f4ea8a Handle SignatureException during package moves Ignore package moves for packages that raise SignatureException, just as they are ignored for packages that have a valid signature. Bug: https://bugs.gentoo.org/922142 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 16 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 2342d571b9..ee574f435f 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -296,8 +296,12 @@ class bindbapi(fakedbapi): encoding_key = True elif binpkg_format == "gpkg": mybinpkg = portage.gpkg.gpkg(self.settings, cpv_str, binpkg_path) -mydata = mybinpkg.get_metadata() -if mybinpkg.signature_exist: +try: +mydata = mybinpkg.get_metadata() +signature_exist = mybinpkg.signature_exist +except SignatureException: +signature_exist = True +if signature_exist: writemsg( colorize( "WARN", @@ -721,8 +725,12 @@ class binarytree: decode_metadata_name = False elif binpkg_format == "gpkg": mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, binpkg_path) -mydata = mybinpkg.get_metadata() -if mybinpkg.signature_exist: +try: +mydata = mybinpkg.get_metadata() +signature_exist = mybinpkg.signature_exist +except SignatureException: +signature_exist = True +if signature_exist: writemsg( colorize( "WARN",
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: c9ed8136f9f0c56dc7b72d400eaa0acbd338778f Author: Konstantin Tokarev yandex ru> AuthorDate: Fri Jan 12 17:47:45 2024 + Commit: Sam James gentoo org> CommitDate: Tue Jan 16 05:16:08 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c9ed8136 vartree: _needs_move() should replace file if filecmp fails for whatever reason When trying to update I've got "OSError: [Errno 5] Input/output error" exception traceback from portage originating from _needs_move(). It turned out that 3 files in my root filesystem were corrupted, which caused filecmp to fail. This patch makes portage replace original file in this case. Bug: https://bugs.gentoo.org/722270 Signed-off-by: Konstantin Tokarev yandex.ru> Closes: https://github.com/gentoo/portage/pull/1233 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 14 +- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 5d39ca1965..a00c731c57 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -6283,7 +6283,19 @@ class dblink: if not _cmpxattr(src_bytes, dest_bytes, exclude=excluded_xattrs): return True -return not filecmp.cmp(src_bytes, dest_bytes, shallow=False) +try: +files_equal = filecmp.cmp(src_bytes, dest_bytes, shallow=False) +except Exception as e: +writemsg( +_( +"Exception '%s' happened when comparing files %s and %s, will replace the latter\n" +) +% (e, mysrc, mydest), +noiselevel=-1, +) +return True + +return not files_equal def merge(
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 58b094bc79e999e44a5b108e2b7273c164aa906e
Author: Alfred Wingate protonmail com>
AuthorDate: Fri Jan 5 19:30:38 2024 +
Commit: Sam James gentoo org>
CommitDate: Tue Jan 16 05:16:09 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=58b094bc
bintree: use urllib provided attributes for hostname, user and password
In addition to simplifying it also solves an issue with parsing raw
ipv6 addresses.
Bug: https://bugs.gentoo.org/921400
Signed-off-by: Alfred Wingate protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1230
Signed-off-by: Sam James gentoo.org>
lib/portage/dbapi/bintree.py | 18 --
1 file changed, 4 insertions(+), 14 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 5fc4716980..2342d571b9 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1344,23 +1344,13 @@ class binarytree:
for repo in reversed(list(self._binrepos_conf.values())):
base_url = repo.sync_uri
parsed_url = urlparse(base_url)
-host = parsed_url.netloc
+host = parsed_url.hostname or ""
port = parsed_url.port
-user = None
-passwd = None
-user_passwd = ""
+user = parsed_url.username
+passwd = parsed_url.password
+user_passwd = user + "@" if user else ""
gpkg_only_warned = False
-if "@" in host:
-user, host = host.split("@", 1)
-user_passwd = user + "@"
-if ":" in user:
-user, passwd = user.split(":", 1)
-
-if port is not None:
-port_str = f":{port}"
-if host.endswith(port_str):
-host = host[: -len(port_str)]
pkgindex_file = os.path.join(
self.settings["EROOT"],
CACHE_PATH,
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/resolver/, lib/_emerge/
commit: 4b885b9ca063c990b1e218c73a786e9d434717e8
Author: Zac Medico gentoo org>
AuthorDate: Mon Jan 8 06:04:37 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jan 8 08:08:51 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4b885b9c
_calc_depclean: add dep_check action
Add a dep_check action which can be used to check the
dependencies of all installed packages. The plan is for depgraph
to use this action to check for broken dependencies prior to the
merge order calculation. The new frozen_config parameter will
allow depgraph to pass a shared frozen config to _calc_depclean.
The result of the dep_check action becomes stale as soon as there
is any change to the installed packages. So, in order to account
for dependencies that may become broken or satisfied during the
process of updating installed packages, the merge order
calculation will need to refresh the dep_check calculation for
every merge order choice that it makes. This refresh will need
to be optimized to identify the portion of the graph that would
become stale due to a given change, so that it can avoid
unnecessary repetition of work.
Bug: https://bugs.gentoo.org/921333
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/actions.py | 21 ++-
lib/_emerge/depgraph.py | 7 ++-
lib/portage/tests/resolver/ResolverPlayground.py | 43 --
lib/portage/tests/resolver/meson.build | 1 +
lib/portage/tests/resolver/test_broken_deps.py | 76
5 files changed, 138 insertions(+), 10 deletions(-)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 20f3978f77..2710c4856c 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -909,7 +909,16 @@ _depclean_result = collections.namedtuple(
)
-def _calc_depclean(settings, trees, ldpath_mtimes, myopts, action, args_set,
spinner):
+def _calc_depclean(
+settings,
+trees,
+ldpath_mtimes,
+myopts,
+action,
+args_set,
+spinner,
+frozen_config=None,
+):
allow_missing_deps = bool(args_set)
debug = "--debug" in myopts
@@ -988,12 +997,14 @@ def _calc_depclean(settings, trees, ldpath_mtimes,
myopts, action, args_set, spi
writemsg_level("\nCalculating dependencies ")
resolver_params = create_depgraph_params(myopts, "remove")
-resolver = depgraph(settings, trees, myopts, resolver_params, spinner)
+resolver = depgraph(
+settings, trees, myopts, resolver_params, spinner,
frozen_config=frozen_config
+)
resolver._load_vdb()
vardb = resolver._frozen_config.trees[eroot]["vartree"].dbapi
real_vardb = trees[eroot]["vartree"].dbapi
-if action == "depclean":
+if action in ("dep_check", "depclean"):
if args_set:
if deselect:
# Start with an empty set.
@@ -1002,6 +1013,7 @@ def _calc_depclean(settings, trees, ldpath_mtimes,
myopts, action, args_set, spi
# Pull in any sets nested within the selected set.
selected_set.update(psets["selected"].getNonAtoms())
+if args_set or action == "dep_check":
# Pull in everything that's installed but not matched
# by an argument atom since we don't want to clean any
# package if something depends on it.
@@ -1098,6 +1110,9 @@ def _calc_depclean(settings, trees, ldpath_mtimes,
myopts, action, args_set, spi
if not success:
return _depclean_result(1, [], False, 0, resolver)
+if action == "dep_check":
+return _depclean_result(0, [], False, 0, resolver)
+
def unresolved_deps():
soname_deps = set()
unresolvable = set()
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index a2865cad23..b859e68224 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -11723,6 +11723,7 @@ def backtrack_depgraph(
myaction: Optional[str],
myfiles: list[str],
spinner: "_emerge.stdout_spinner.stdout_spinner",
+frozen_config: Optional[_frozen_depgraph_config] = None,
) -> tuple[Any, depgraph, list[str]]:
"""
@@ -11747,6 +11748,7 @@ def _backtrack_depgraph(
myaction: Optional[str],
myfiles: list[str],
spinner: "_emerge.stdout_spinner.stdout_spinner",
+frozen_config: Optional[_frozen_depgraph_config] = None,
) -> tuple[Any, depgraph, list[str], int, int]:
debug = "--debug" in myopts
mydepgraph = None
@@ -11756,7 +11758,10 @@ def _backtrack_depgraph(
backtracker = Backtracker(max_depth)
backtracked = 0
-frozen_config = _frozen_depgraph_config(settings, trees, myopts, myparams,
spinner)
+if frozen_config is None:
+frozen_config = _frozen_depgraph_config(
+settings, trees, myopts, myparams, spinner
+)
while backtracker:
if debug and mydepgraph is not None:
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
