[gentoo-commits] proj/portage:master commit in: /, lib/portage/util/
commit: 38ff9e6ce8625d4d2a993a9aa3ee488043e8210e
Author: Mike Gilbert gentoo org>
AuthorDate: Mon Sep 23 16:15:30 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Mon Sep 23 16:29:38 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=38ff9e6c
Remove obsolete pylint options
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/whirlpool.py | 2 --
pylintrc | 7 ---
2 files changed, 9 deletions(-)
diff --git a/lib/portage/util/whirlpool.py b/lib/portage/util/whirlpool.py
index 62fcfda532..4726846ffe 100644
--- a/lib/portage/util/whirlpool.py
+++ b/lib/portage/util/whirlpool.py
@@ -28,8 +28,6 @@
##
# This Python implementation is therefore also placed in the public domain.
-# pylint: disable=mixed-indentation
-
import warnings
from portage.localization import _
diff --git a/pylintrc b/pylintrc
index 9d3dae6212..612f967e22 100644
--- a/pylintrc
+++ b/pylintrc
@@ -219,13 +219,6 @@ max-line-length=100
# Maximum number of lines in a module.
max-module-lines=1
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,
- dict-separator
-
# Allow the body of a class to be on the same line as the declaration if body
# contains single statement.
single-line-class-stmt=no
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 492506adede9d96c661699b90295b70e50f30160
Author: Ulrich Müller gentoo org>
AuthorDate: Thu Jun 20 06:02:31 2024 +
Commit: Ulrich Müller gentoo org>
CommitDate: Mon Sep 9 18:04:44 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=492506ad
eapi.py: Use attrs instead of hardcoding EAPIs in functions
Adding new attrs as needed. Their name is the same as the
corresponding PMS feature label (if one exists).
Signed-off-by: Ulrich Müller gentoo.org>
lib/portage/eapi.py | 30 --
1 file changed, 24 insertions(+), 6 deletions(-)
diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py
index ee691d311d..86b27bdbc5 100644
--- a/lib/portage/eapi.py
+++ b/lib/portage/eapi.py
@@ -41,7 +41,7 @@ def eapi_has_strong_blocks(eapi: str) -> bool:
def eapi_has_src_prepare_and_src_configure(eapi: str) -> bool:
-return eapi not in ("0", "1")
+return _get_eapi_attrs(eapi).src_prepare_src_configure
def eapi_supports_prefix(eapi: str) -> bool:
@@ -77,15 +77,15 @@ def eapi_exports_ECLASSDIR(eapi: str) -> bool:
def eapi_has_pkg_pretend(eapi: str) -> bool:
-return eapi not in ("0", "1", "2", "3")
+return _get_eapi_attrs(eapi).pkg_pretend
def eapi_has_implicit_rdepend(eapi: str) -> bool:
-return eapi in ("0", "1", "2", "3")
+return _get_eapi_attrs(eapi).rdepend_depend
def eapi_has_dosed_dohard(eapi: str) -> bool:
-return eapi in ("0", "1", "2", "3")
+return _get_eapi_attrs(eapi).dosed_dohard
def eapi_has_required_use(eapi: str) -> bool:
@@ -109,11 +109,11 @@ def eapi_has_repo_deps(eapi: str) -> bool:
def eapi_supports_stable_use_forcing_and_masking(eapi: str) -> bool:
-return eapi not in ("0", "1", "2", "3", "4", "4-slot-abi")
+return _get_eapi_attrs(eapi).stablemask
def eapi_allows_directories_on_profile_level_and_repository_level(eapi: str)
-> bool:
-return eapi not in ("0", "1", "2", "3", "4", "4-slot-abi", "5", "6")
+return _get_eapi_attrs(eapi).profile_file_dirs
def eapi_allows_package_provided(eapi: str) -> bool:
@@ -150,6 +150,7 @@ _eapi_attrs = collections.namedtuple(
"allows_package_provided",
"bdepend",
"broot",
+"dosed_dohard",
"empty_groups_always_true",
"exports_AA",
"exports_EBUILD_PHASE_FUNC",
@@ -164,14 +165,19 @@ _eapi_attrs = collections.namedtuple(
"iuse_effective",
"posixish_locale",
"path_variables_end_with_trailing_slash",
+"pkg_pretend",
"prefix",
+"profile_file_dirs",
+"rdepend_depend",
"repo_deps",
"required_use",
"required_use_at_most_one_of",
"selective_src_uri_restriction",
"slot_operator",
"slot_deps",
+"src_prepare_src_configure",
"src_uri_arrows",
+"stablemask",
"strong_blocks",
"sysroot",
"use_deps",
@@ -223,6 +229,7 @@ def _get_eapi_attrs(eapi_str: Optional[str]) -> _eapi_attrs:
allows_package_provided=True,
bdepend=False,
broot=True,
+dosed_dohard=False,
empty_groups_always_true=False,
exports_AA=False,
exports_EBUILD_PHASE_FUNC=True,
@@ -236,15 +243,20 @@ def _get_eapi_attrs(eapi_str: Optional[str]) ->
_eapi_attrs:
iuse_defaults=True,
iuse_effective=False,
path_variables_end_with_trailing_slash=False,
+pkg_pretend=True,
posixish_locale=False,
prefix=True,
+profile_file_dirs=False,
+rdepend_depend=False,
repo_deps=True,
required_use=True,
required_use_at_most_one_of=True,
selective_src_uri_restriction=True,
slot_deps=True,
slot_operator=True,
+src_prepare_src_configure=True,
src_uri_arrows=True,
+stablemask=True,
strong_blocks=True,
sysroot=True,
use_deps=True,
@@ -256,6 +268,7 @@ def _get_eapi_attrs(eapi_str: Optional[str]) -> _eapi_attrs:
allows_package_provided=eapi <= Eapi("6"),
bdepend=eapi >= Eapi("7"),
broot=eapi >= Eapi("7"),
+dosed_dohard=eapi <= Eapi("3"),
empty_groups_always_true=eapi <= Eapi("6"),
exports_AA=eapi <= Eapi("3"),
exports_EBUILD_PHASE_FUNC=eapi >= Eapi("5"),
@@ -269,15 +282,20 @@ def _get_eapi_attrs(eapi_str: Optional[str]) ->
_eapi_attrs:
iuse_defaults=eapi >= Eapi("1"),
iuse_effective=eapi >= Eapi("5"),
path_variables_end_with_trailing_slash=eapi <= Eapi("6"),
+pkg_pretend=eapi >= Eapi("4"),
posixish_locale=eapi >= Eapi("6"),
prefix=eapi >= Eapi("3"),
+profile_file_dirs=eapi >= Eapi("7"),
+rdepend_depend=eapi <= Eapi("3"),
repo_deps=Fa
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: e5569b2dbd97ee804ee37ec17c74ae64ec8826b6 Author: Ulrich Müller gentoo org> AuthorDate: Thu Jun 20 06:01:36 2024 + Commit: Ulrich Müller gentoo org> CommitDate: Mon Sep 9 18:04:38 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e5569b2d eapi.py: Sort _eapi_attrs alphabetically Signed-off-by: Ulrich Müller gentoo.org> lib/portage/eapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py index 2c17018709..ee691d311d 100644 --- a/lib/portage/eapi.py +++ b/lib/portage/eapi.py @@ -150,6 +150,7 @@ _eapi_attrs = collections.namedtuple( "allows_package_provided", "bdepend", "broot", +"empty_groups_always_true", "exports_AA", "exports_EBUILD_PHASE_FUNC", "exports_ECLASSDIR", @@ -172,10 +173,9 @@ _eapi_attrs = collections.namedtuple( "slot_deps", "src_uri_arrows", "strong_blocks", +"sysroot", "use_deps", "use_dep_defaults", -"empty_groups_always_true", -"sysroot", ), )
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: b30ddb1913e8aa2947d20e43f455d2060aa6257f
Author: Zac Medico gentoo org>
AuthorDate: Sat Aug 31 20:08:35 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Sep 1 06:59:44 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b30ddb19
asyncio: Avoid _wrap_loop prior to create_subprocess_exec
Bug: https://bugs.gentoo.org/761538
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/_asyncio/__init__.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index 48d9b68104..4cf337998c 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -119,7 +119,7 @@ def run(coro):
run.__doc__ = _real_asyncio.run.__doc__
-def create_subprocess_exec(*args, **kwargs):
+def create_subprocess_exec(*args, loop=None, **kwargs):
"""
Create a subprocess.
@@ -140,7 +140,6 @@ def create_subprocess_exec(*args, **kwargs):
@rtype: asyncio.subprocess.Process (or compatible)
@return: asyncio.subprocess.Process interface
"""
-loop = _wrap_loop(kwargs.pop("loop", None))
# Python 3.4 and later implement PEP 446, which makes newly
# created file descriptors non-inheritable by default.
kwargs.setdefault("close_fds", False)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: 7f6b2b04878209130d44fc06105f521bae2b2173 Author: Zac Medico gentoo org> AuthorDate: Sat Aug 31 05:35:32 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Sep 1 06:58:56 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7f6b2b04 asyncio: Use default sleep implementation when possible When a loop argument is not given, use the default asyncio sleep implementation and avoid unnecessary _wrap_loop usage. Bug: https://bugs.gentoo.org/761538 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/_asyncio/__init__.py | 7 +-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index 23c664e763..a235d87246 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -210,9 +210,12 @@ def sleep(delay, result=None, loop=None): @param result: result of the future @type loop: asyncio.AbstractEventLoop (or compatible) @param loop: event loop -@rtype: asyncio.Future (or compatible) -@return: an instance of Future +@rtype: collections.abc.Coroutine or asyncio.Future +@return: an instance of Coroutine or Future """ +if loop is None: +return _real_asyncio.sleep(delay, result=result) + loop = _wrap_loop(loop) future = loop.create_future() handle = loop.call_later(delay, future.set_result, result)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: e5457915f7929db3781ded384bdb089b0760221f Author: Zac Medico gentoo org> AuthorDate: Sat Aug 31 17:32:12 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Sep 1 06:59:34 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e5457915 asyncio: Use default ensure_future implementation when possible When a loop argument is not given, use the default asyncio ensure_future implementation and avoid unnecessary _wrap_loop usage. Bug: https://bugs.gentoo.org/761538 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/_asyncio/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index a235d87246..48d9b68104 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -186,6 +186,9 @@ def ensure_future(coro_or_future, loop=None): @rtype: asyncio.Future (or compatible) @return: an instance of Future """ +if loop is None: +return _real_asyncio.ensure_future(coro_or_future) + loop = _wrap_loop(loop) if isinstance(loop._asyncio_wrapper, _AsyncioEventLoop): # Use the real asyncio loop and ensure_future.
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: ee17cbd807ba976491e4c657be8aa9b9a29fe059
Author: Zac Medico gentoo org>
AuthorDate: Sat Aug 31 19:06:25 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Aug 31 19:06:25 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ee17cbd8
_safe_loop: Discard wrapped asyncio.run loop that was closed
Since commit cb0c09d8cecb, _get_running_loop can wrap loops from
asyncio.run, so these loops need to be discarded if they've been
closed.
Fixes: cb0c09d8cecb ("Support coroutine exitfuncs for non-main loops")
Bug: https://bugs.gentoo.org/938761
Bug: https://bugs.gentoo.org/761538
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/_asyncio/__init__.py | 7 +++
1 file changed, 7 insertions(+)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index bdacda59ce..23c664e763 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -311,6 +311,13 @@ def _safe_loop(create: Optional[bool] = True) ->
Optional[_AsyncioEventLoop]:
_thread_weakrefs.loops = weakref.WeakValueDictionary()
try:
loop = _thread_weakrefs.loops[thread_key]
+if loop.is_closed():
+# Discard wrapped asyncio.run loop that was closed.
+del _thread_weakrefs.loops[thread_key]
+if loop is _thread_weakrefs.mainloop:
+_thread_weakrefs.mainloop = None
+loop = None
+raise KeyError(thread_key)
except KeyError:
if not create:
return None
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: a62faf99dbd0078cd58a76e6419e0a2d0d14d636
Author: Zac Medico gentoo org>
AuthorDate: Sun Aug 18 15:00:07 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Aug 18 15:47:01 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a62faf99
Remove unused and unimplemented asyncio.Task class
This class originated from commit 142d08c0636b and it is unused
since _PortageEventLoop was removed in commit 20204fd8c29.
Fixes: 20204fd8c29 ("Remove unused _PortageEventLoop and _PortageChildWatcher")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/_asyncio/__init__.py | 11 ---
1 file changed, 11 deletions(-)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index c960d03630..bdacda59ce 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -19,7 +19,6 @@ __all__ = (
"run",
"shield",
"sleep",
-"Task",
"wait",
"wait_for",
)
@@ -174,16 +173,6 @@ class Lock(_Lock):
super().__init__(**kwargs)
-class Task(Future):
-"""
-Schedule the execution of a coroutine: wrap it in a future. A task
-is a subclass of Future.
-"""
-
-def __init__(self, coro, loop=None):
-raise NotImplementedError
-
-
def ensure_future(coro_or_future, loop=None):
"""
Wrap a coroutine or an awaitable in a future.
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: e97acd3c62ff02eb41ff643e75eb5e07c27717f8
Author: Zac Medico gentoo org>
AuthorDate: Sun Aug 18 14:59:46 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Aug 18 15:46:45 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e97acd3c
_wrap_loop: Prevent redundant AsyncioEventLoop instances
Ultimately the loop arguments that necessitate the _wrap_loop
function can be removed, because our aim since bug 761538 should
be to eliminate them. Meanwhile, we don't want _wrap_loop to return
redundant AsyncioEventLoop instances if we can easily prevent it.
Therefore, use _safe_loop(create=False) to look up the AsyncioEventLoop
instance associated with the current thread, and avoid creating
redundant instances. This serves to guard against garbage collection
of AsyncioEventLoop instances which may have _coroutine_exithandlers
added by the atexit_register function since commit cb0c09d8cecb from
bug 937740.
If _safe_loop(create=False) fails to associate a loop with the current
thread, raise an AssertionError for portage internal API consumers.
It's not known whether external API consumers will trigger this case,
so if it happens then emit a UserWarning and return a temporary
AsyncioEventLoop instance.
Fixes: cb0c09d8cecb ("Support coroutine exitfuncs for non-main loops")
Bug: https://bugs.gentoo.org/938127
Bug: https://bugs.gentoo.org/937740
Bug: https://bugs.gentoo.org/761538
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/_asyncio/__init__.py | 43 +++
1 file changed, 38 insertions(+), 5 deletions(-)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index 8805e35756..c960d03630 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -26,6 +26,7 @@ __all__ = (
import sys
import types
+import warnings
import weakref
import asyncio as _real_asyncio
@@ -46,6 +47,7 @@ from asyncio import (
)
import threading
+from typing import Optional
import portage
@@ -251,11 +253,35 @@ def _wrap_loop(loop=None):
# The default loop returned by _wrap_loop should be consistent
# with global_event_loop, in order to avoid accidental registration
# of callbacks with a loop that is not intended to run.
-loop = loop or _safe_loop()
-return loop if hasattr(loop, "_asyncio_wrapper") else
_AsyncioEventLoop(loop=loop)
+if hasattr(loop, "_asyncio_wrapper"):
+return loop
+
+# This returns a running loop if it exists, and otherwise returns
+# a loop associated with the current thread.
+safe_loop = _safe_loop(create=loop is None)
+if safe_loop is not None and (loop is None or safe_loop._loop is loop):
+return safe_loop
+
+if safe_loop is None:
+msg = f"_wrap_loop argument '{loop}' not associated with thread
'{threading.get_ident()}'"
+else:
+msg = f"_wrap_loop argument '{loop}' different frome loop
'{safe_loop._loop}' already associated with thread '{threading.get_ident()}'"
+
+if portage._internal_caller:
+raise AssertionError(msg)
+# It's not known whether external API consumers will trigger this case,
+# so if it happens then emit a UserWarning before returning a temporary
+# AsyncioEventLoop instance.
+warnings.warn(msg, UserWarning, stacklevel=2)
-def _safe_loop():
+# We could possibly add a weak reference in _thread_weakrefs.loops when
+# safe_loop is None, but if safe_loop is not None, then there is a
+# collision in _thread_weakrefs.loops that would need to be resolved.
+return _AsyncioEventLoop(loop=loop)
+
+
+def _safe_loop(create: Optional[bool] = True) -> Optional[_AsyncioEventLoop]:
"""
Return an event loop that's safe to use within the current context.
For portage internal callers or external API consumers calling from
@@ -276,8 +302,13 @@ def _safe_loop():
are added to a WeakValueDictionary, and closed via an atexit hook
if they still exist during exit for the current pid.
-@rtype: asyncio.AbstractEventLoop (or compatible)
-@return: event loop instance
+@type create: bool
+@param create: Create a loop by default if a loop is not already associated
+with the current thread. If create is False, then return None if a loop
+is not already associated with the current thread.
+@rtype: AsyncioEventLoop or None
+@return: event loop instance, or None if the create parameter is False and
+a loop is not already associated with the current thread.
"""
loop = _get_running_loop()
if loop is not None:
@@ -292,6 +323,8 @@ def _safe_loop():
try:
loop = _thread_weakrefs.loops[thread_key]
except KeyError:
+if not create:
+return None
try:
try:
_loop = _real_asyncio.get_running_loop()
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 5ee1a193982fce006aefbd5a6c5907392016b44d
Author: mid-kid gmail com>
AuthorDate: Sat Aug 3 15:04:43 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Thu Aug 15 17:29:00 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5ee1a193
Make portage.util.compression_probe work when ctypes is unavailable
This is useful for bootstrapping purposes, as _ctypes requires a dynamic
linker.
Closes: https://github.com/gentoo/portage/pull/1363
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/compression_probe.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/portage/util/compression_probe.py
b/lib/portage/util/compression_probe.py
index 0879754b21..268e5840cc 100644
--- a/lib/portage/util/compression_probe.py
+++ b/lib/portage/util/compression_probe.py
@@ -1,13 +1,13 @@
# Copyright 2015-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import ctypes
import errno
import re
from portage import _encodings, _unicode_encode
from portage.exception import FileNotFound, PermissionDenied
+from portage.util._ctypes import ctypes
_compressors = {
"bzip2": {
@@ -49,7 +49,7 @@ _compressors = {
# if the current architecture can support it, which is true when
# sizeof(long) is at least 8 bytes.
"decompress": "zstd -d"
-+ (" --long=31" if ctypes.sizeof(ctypes.c_long) >= 8 else ""),
++ (" --long=31" if ctypes and ctypes.sizeof(ctypes.c_long) >= 8 else
""),
"package": "app-arch/zstd",
},
}
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: cfd767cd35f5affd3b61b665b0f8814fe2de24c4 Author: Zac Medico gentoo org> AuthorDate: Wed Aug 14 05:30:42 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Aug 14 15:22:05 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cfd767cd run_exitfuncs: Drop hooks inherited via fork Drop hooks inherited via fork because they can trigger redundant actions as shown in bug 937891. Note that atexit hooks only work after fork since issue 83856 was fixed in Python 3.13. Bug: https://bugs.gentoo.org/937891 Signed-off-by: Zac Medico gentoo.org> lib/portage/process.py | 9 +++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/portage/process.py b/lib/portage/process.py index 38adebda66..e6f6feb357 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -208,7 +208,7 @@ def atexit_register(func, *args, **kargs): # which is associated with the current thread. global_event_loop()._coroutine_exithandlers.append((func, args, kargs)) else: -_exithandlers.append((func, args, kargs)) +_exithandlers.append((func, args, kargs, portage.getpid())) def run_exitfuncs(): @@ -222,7 +222,12 @@ def run_exitfuncs(): # original function is in the output to stderr. exc_info = None while _exithandlers: -func, targs, kargs = _exithandlers.pop() +func, targs, kargs, pid = _exithandlers.pop() +if pid != portage.getpid(): +# Drop hooks inherited via fork because they can trigger redundant +# actions as shown in bug 937891. Note that atexit hooks only work +# after fork since issue 83856 was fixed in Python 3.13. +continue try: func(*targs, **kargs) except SystemExit:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/_eventloop/, ...
commit: cb0c09d8cecbcc086786e3e2c7cdd8ffc023a48a
Author: Zac Medico gentoo org>
AuthorDate: Sun Aug 11 07:50:49 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Aug 11 07:50:49 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cb0c09d8
Support coroutine exitfuncs for non-main loops
Since an API consumer can cause loops to be instantiated
for non-main threads, support coroutine exitfuncs for each
loop. The included Socks5ServerAtExitThreadedTestCase calls
get_socks5_proxy from a non-main thread, and demonstrates
that coroutine exitfuncs for the resulting non-main loop
will reliably stop the socks5 proxy via atexit hook.
The _thread_weakrefs_atexit function will now make a
temporary adjustment to _thread_weakrefs.loops so that a
loop is associated with the current thread when it is
closing. Also, the _get_running_loop function will now
store weak references to all _AsyncioEventLoop instances
it creates, since each has a _coroutine_exithandlers
attribute that can be modified by atexit_register calls.
Bug: https://bugs.gentoo.org/937740
Signed-off-by: Zac Medico gentoo.org>
lib/portage/process.py| 11 --
lib/portage/tests/util/test_socks5.py | 38 +++--
lib/portage/util/_eventloop/asyncio_event_loop.py | 15 ++---
lib/portage/util/futures/_asyncio/__init__.py | 41 ---
4 files changed, 76 insertions(+), 29 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 23e2507b53..38adebda66 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -194,7 +194,6 @@ def spawn_fakeroot(mycommand, fakeroot_state=None,
opt_name=None, **keywords):
_exithandlers = []
-_coroutine_exithandlers = []
def atexit_register(func, *args, **kargs):
@@ -205,7 +204,9 @@ def atexit_register(func, *args, **kargs):
# The internal asyncio wrapper module would trigger a circular import
# if used here.
if _asyncio.iscoroutinefunction(func):
-_coroutine_exithandlers.append((func, args, kargs))
+# Add this coroutine function to the exit handlers for the loop
+# which is associated with the current thread.
+global_event_loop()._coroutine_exithandlers.append((func, args, kargs))
else:
_exithandlers.append((func, args, kargs))
@@ -238,13 +239,17 @@ async def run_coroutine_exitfuncs():
"""
This is the same as run_exitfuncs but it uses asyncio.iscoroutinefunction
to check which functions to run. It is called by the AsyncioEventLoop
-_close_main method just before the loop is closed.
+_close method just before the loop is closed.
If the loop is explicitly closed before exit, then that will cause
run_coroutine_exitfuncs to run before run_exitfuncs. Otherwise, a
run_exitfuncs hook will close it, causing run_coroutine_exitfuncs to be
called via run_exitfuncs.
"""
+# The _thread_weakrefs_atexit function makes an adjustment to ensure
+# that global_event_loop() returns the correct loop when it is closing,
+# regardless of which thread the loop was initially associated with.
+_coroutine_exithandlers = global_event_loop()._coroutine_exithandlers
tasks = []
while _coroutine_exithandlers:
func, targs, kargs = _coroutine_exithandlers.pop()
diff --git a/lib/portage/tests/util/test_socks5.py
b/lib/portage/tests/util/test_socks5.py
index 35f919d970..078e3b1a23 100644
--- a/lib/portage/tests/util/test_socks5.py
+++ b/lib/portage/tests/util/test_socks5.py
@@ -194,17 +194,17 @@ class Socks5ServerTestCase(TestCase):
asyncio.run(self._test_socks5_proxy())
async def _test_socks5_proxy(self):
-loop = asyncio.get_running_loop()
+loop = global_event_loop()
host = "127.0.0.1"
content = b"Hello World!"
path = "/index.html"
proxy = None
tempdir = tempfile.mkdtemp()
-previous_exithandlers = portage.process._coroutine_exithandlers
+previous_exithandlers = loop._coroutine_exithandlers
try:
-portage.process._coroutine_exithandlers = []
+loop._coroutine_exithandlers = []
with AsyncHTTPServer(host, {path: content}, loop) as server:
settings = {
"PORTAGE_TMPDIR": tempdir,
@@ -227,11 +227,11 @@ class Socks5ServerTestCase(TestCase):
finally:
try:
# Also run_coroutine_exitfuncs to test atexit hook cleanup.
-self.assertNotEqual(portage.process._coroutine_exithandlers,
[])
+self.assertNotEqual(loop._coroutine_exithandlers, [])
await portage.process.run_coroutine_exitfuncs()
-self.assertEqual(portage.process._coroutine_exithandlers, [])
+self.assertEqual(loop._coroutine_exithandlers, [])
finally:
-portage.process._coroutine_e
[gentoo-commits] proj/portage:master commit in: lib/portage/, lib/portage/util/_async/, lib/portage/tests/, ...
commit: d6710ee0cdab2a212ff70503f9699f1be4660bb4
Author: Zac Medico gentoo org>
AuthorDate: Thu Aug 8 14:58:31 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Aug 8 14:58:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d6710ee0
run_exitfuncs: Support loop close via hook
Handle the case where the loop has not been explicitly closed before
exit. In this case, run_exitfuncs previously tried to call coroutine
functions as though they were normal functions, which obvously would
not behave correctly.
Solve this problem by storing the coroutine functions in a separate
_coroutine_exithandlers list that belongs exclusively to the
run_coroutine_exitfuncs function, so that it is safe to close the
loop and call run_coroutine_exitfuncs from inside a run_exitfuncs
hook. A _thread_weakrefs_atexit hook already exists that will close
weakly referenced loops. The _thread_weakrefs_atexit hook is now
fixed to release its lock when it closes a loop, since the same
lock may need to be re-acquired when run_coroutine_exitfuncs runs.
The included test case demonstrates that run_exitfuncs will run
via an atexit hook and correctly terminate the socks5 proxy in a
standalone program using the portage API (like eclean).
Due to a deadlock that will occur if an _exit_function atexit hook
from the multiprocessing module executes before run_exitfuncs, a
portage.process._atexit_register_run_exitfuncs() function needs to
be called in order to re-order the hooks after the first process
has been started via the multiprocessing module. The natural place
to call this is in the ForkProcess class, using a global variable
to trigger the call just once.
Fixes: c3ebdbb42e72 ("elog/mod_custom: Spawn processes in background")
Bug: https://bugs.gentoo.org/937384
Signed-off-by: Zac Medico gentoo.org>
lib/portage/process.py| 37 ---
lib/portage/tests/__init__.py | 19
lib/portage/tests/util/test_socks5.py | 66 ---
lib/portage/util/_async/ForkProcess.py| 8
lib/portage/util/futures/_asyncio/__init__.py | 24 ++
5 files changed, 133 insertions(+), 21 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 6e4e0d7162..23e2507b53 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -3,6 +3,7 @@
# Distributed under the terms of the GNU General Public License v2
+import asyncio as _asyncio
import atexit
import errno
import fcntl
@@ -193,6 +194,7 @@ def spawn_fakeroot(mycommand, fakeroot_state=None,
opt_name=None, **keywords):
_exithandlers = []
+_coroutine_exithandlers = []
def atexit_register(func, *args, **kargs):
@@ -200,7 +202,12 @@ def atexit_register(func, *args, **kargs):
what is registered. For example, when portage restarts itself via
os.execv, the atexit module does not work so we have to do it
manually by calling the run_exitfuncs() function in this module."""
-_exithandlers.append((func, args, kargs))
+# The internal asyncio wrapper module would trigger a circular import
+# if used here.
+if _asyncio.iscoroutinefunction(func):
+_coroutine_exithandlers.append((func, args, kargs))
+else:
+_exithandlers.append((func, args, kargs))
def run_exitfuncs():
@@ -232,12 +239,16 @@ async def run_coroutine_exitfuncs():
This is the same as run_exitfuncs but it uses asyncio.iscoroutinefunction
to check which functions to run. It is called by the AsyncioEventLoop
_close_main method just before the loop is closed.
+
+If the loop is explicitly closed before exit, then that will cause
+run_coroutine_exitfuncs to run before run_exitfuncs. Otherwise, a
+run_exitfuncs hook will close it, causing run_coroutine_exitfuncs to be
+called via run_exitfuncs.
"""
tasks = []
-for index, (func, targs, kargs) in
reversed(list(enumerate(_exithandlers))):
-if asyncio.iscoroutinefunction(func):
-del _exithandlers[index]
-tasks.append(asyncio.ensure_future(func(*targs, **kargs)))
+while _coroutine_exithandlers:
+func, targs, kargs = _coroutine_exithandlers.pop()
+tasks.append(asyncio.ensure_future(func(*targs, **kargs)))
tracebacks = []
exc_info = None
for task in tasks:
@@ -255,7 +266,21 @@ async def run_coroutine_exitfuncs():
raise exc_info[1].with_traceback(exc_info[2])
-atexit.register(run_exitfuncs)
+def _atexit_register_run_exitfuncs():
+"""
+Register the run_exitfuncs atexit hook. If this hook is not called
+before the multiprocessing module's _exit_function, then there will
+be a deadlock. In order to prevent the deadlock, this function must
+be called in order to re-order the hooks after the first process has
+been started via the multiprocessing module. The natural place to
+call this is in the ForkProcess class, though it should
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/
commit: 9e6451c88e3da11e0eb7b0bd6b1497c5ca4fb67f
Author: Zac Medico gentoo org>
AuthorDate: Tue Aug 6 04:48:26 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Aug 7 14:39:25 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9e6451c8
doebuild.spawn: Skip socks5 proxy for "depend" phase
Skip the socks5 proxy for the "depend" phase. It should not be needed
because we only allow bash builtin commands during this phase.
Since the socks5 proxy requires portage's event loop to be explictly
closed before exit, skipping it will allow programs like eclean-dist
to avoid the need to explicitly close portage's event loop before exit.
Bug: https://bugs.gentoo.org/937384
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/doebuild.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 403836b80b..b5fb46df70 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -2009,7 +2009,7 @@ def spawn(
if (
not networked
-and mysettings.get("EBUILD_PHASE") != "nofetch"
+and mysettings.get("EBUILD_PHASE") not in ("depend", "nofetch")
and ("network-sandbox-proxy" in features or "distcc" in features)
):
# Provide a SOCKS5-over-UNIX-socket proxy to escape sandbox
[gentoo-commits] proj/portage:master commit in: lib/portage/util/elf/, lib/portage/dep/soname/
commit: 1ea8bb30da1ca9a47fd795d2fa1c829fda95bfec
Author: Sam James gentoo org>
AuthorDate: Wed Aug 7 11:56:17 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Aug 7 11:56:17 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1ea8bb30
ELF: add entries for BPF
Bug: https://bugs.gentoo.org/937485
Signed-off-by: Sam James gentoo.org>
lib/portage/dep/soname/multilib_category.py | 4 +++-
lib/portage/util/elf/constants.py | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dep/soname/multilib_category.py
b/lib/portage/dep/soname/multilib_category.py
index baca439fd2..e85191968d 100644
--- a/lib/portage/dep/soname/multilib_category.py
+++ b/lib/portage/dep/soname/multilib_category.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2019 Gentoo Authors
+# Copyright 2015-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
#
# Compute a multilib category, as discussed here:
@@ -59,6 +59,7 @@ from portage.util.elf.constants import (
EM_ARC_COMPACT3_64,
EM_ARM,
EM_ALTERA_NIOS2,
+EM_BPF,
EM_IA_64,
EM_LOONGARCH,
EM_MIPS,
@@ -91,6 +92,7 @@ _machine_prefix_map = {
EM_ARC_COMPACT3: "arc",
EM_ARC_COMPACT3_64: "arc",
EM_ARM: "arm",
+EM_BPF: "bpf",
EM_IA_64: "ia64",
EM_LOONGARCH: "loong",
EM_MIPS: "mips",
diff --git a/lib/portage/util/elf/constants.py
b/lib/portage/util/elf/constants.py
index 9216a35353..e389d1292a 100644
--- a/lib/portage/util/elf/constants.py
+++ b/lib/portage/util/elf/constants.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2019 Gentoo Authors
+# Copyright 2015-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
#
# These constants are available from elfutils:
@@ -40,6 +40,7 @@ EM_AARCH64 = 183
EM_ARC_COMPACT2 = 195
EM_AMDGPU = 224
EM_RISCV = 243
+EM_BPF = 247
EM_ARC_COMPACT3_64 = 253
EM_ARC_COMPACT3 = 255
EM_LOONGARCH = 258
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/tests/ebuild/, /, ...
commit: 8b5b5186965c47605ba004d317e8fd58e70e97cd
Author: Zac Medico gentoo org>
AuthorDate: Sat Aug 3 21:33:41 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Aug 3 21:33:41 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8b5b5186
_EbuildFetcherProcess: Handle SIGTERM
Fix _EbuildFetcherProcess to handle SIGTERM, so that FETCHCOMMAND
processes will not be left running in the background:
* Convert the fetch function to an async_fetch coroutine function
so that it can use asyncio.CancelledError handlers to terminate
running processes.
* Use multiprocessing.active_children() to detect and terminate
any processes that asyncio.CancelledError handlers did not have
an opportunity to terminate because the exception arrived too
soon after fork/spawn.
* Add unit test to verify that a child process is correctly
killed when EbuildFetcher is cancelled, with short timeout in
case it takes some time for the process to disappear.
Bug: https://bugs.gentoo.org/936273
Signed-off-by: Zac Medico gentoo.org>
NEWS | 3 +
lib/_emerge/EbuildFetcher.py | 68 ++
lib/portage/package/ebuild/fetch.py| 102 -
lib/portage/tests/ebuild/test_fetch.py | 100 +++-
lib/portage/tests/util/test_socks5.py | 16 --
5 files changed, 257 insertions(+), 32 deletions(-)
diff --git a/NEWS b/NEWS
index 04ce6069db..3b71349508 100644
--- a/NEWS
+++ b/NEWS
@@ -28,6 +28,9 @@ Bug fixes:
* repository: config: Allow a repository to be configured using one of its
aliases rather than its primary name (bug #935830).
+* emerge: Fix parallel-fetch to properly terminate FETCOMMAND processes when
+ needed, using a SIGTERM handler (bug #936273).
+
portage-3.0.65 (2024-06-04)
--
diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py
index 81d4b1054b..994271236c 100644
--- a/lib/_emerge/EbuildFetcher.py
+++ b/lib/_emerge/EbuildFetcher.py
@@ -4,6 +4,8 @@
import copy
import functools
import io
+import multiprocessing
+import signal
import sys
import portage
@@ -17,11 +19,12 @@ from portage.package.ebuild.fetch import (
_check_distfile,
_drop_privs_userfetch,
_want_userfetch,
-fetch,
+async_fetch,
)
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.ForkProcess import ForkProcess
from portage.util._pty import _create_pty_or_pipe
+from portage.util.futures import asyncio
from _emerge.CompositeTask import CompositeTask
@@ -34,6 +37,7 @@ class EbuildFetcher(CompositeTask):
"logfile",
"pkg",
"prefetch",
+"pre_exec",
"_fetcher_proc",
)
@@ -253,6 +257,7 @@ class _EbuildFetcherProcess(ForkProcess):
self._get_manifest(),
self._uri_map,
self.fetchonly,
+self.pre_exec,
)
ForkProcess._start(self)
@@ -263,7 +268,10 @@ class _EbuildFetcherProcess(ForkProcess):
self._settings = None
@staticmethod
-def _target(settings, manifest, uri_map, fetchonly):
+def _target(settings, manifest, uri_map, fetchonly, pre_exec):
+if pre_exec is not None:
+pre_exec()
+
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
portage.output.havecolor = settings.get("NOCOLOR") not in ("yes",
"true")
@@ -273,17 +281,53 @@ class _EbuildFetcherProcess(ForkProcess):
if _want_userfetch(settings):
_drop_privs_userfetch(settings)
-rval = 1
allow_missing = manifest.allow_missing or "digest" in settings.features
-if fetch(
-uri_map,
-settings,
-fetchonly=fetchonly,
-digests=copy.deepcopy(manifest.getTypeDigests("DIST")),
-allow_missing_digests=allow_missing,
-):
-rval = os.EX_OK
-return rval
+
+async def main():
+loop = asyncio.get_event_loop()
+task = asyncio.ensure_future(
+async_fetch(
+uri_map,
+settings,
+fetchonly=fetchonly,
+digests=copy.deepcopy(manifest.getTypeDigests("DIST")),
+allow_missing_digests=allow_missing,
+)
+)
+
+def sigterm_handler(signum, _frame):
+loop.call_soon_threadsafe(task.cancel)
+signal.signal(signal.SIGTERM, signal.SIG_IGN)
+
+signal.signal(signal.SIGTERM, sigterm_handler)
+try:
+await task
+except asyncio.CancelledError:
+# If asyncio.CancelledError arrives too soon after fork/spawn
+# then handers will not have an opportunity to terminate
+# the
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/
commit: 45fceb11558e3363390a4b58ab067603b418773e
Author: Eli Schwartz gentoo org>
AuthorDate: Mon Jul 8 03:45:05 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Jul 19 05:41:19 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=45fceb11
ebuild: try to make maintainer mode checks a bit less prone to FP
We sometimes get weird results. For example, in binutils/gdb, we get:
```
* QA Notice: Automake "maintainer mode" detected:
*
* checking for aclocal... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing aclocal-1.15
* checking for autoconf... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing autoconf
* checking for autoheader... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing autoheader
* checking for aclocal... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing aclocal-1.15
* checking for autoconf... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing autoconf
* checking for autoheader... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing autoheader
```
The current pattern looks for the definition of the "missing" script
followed by known autotools commands. With a bit more effort, we can
hopefully match the actual `make` commands that get produced as
maintainer targets. As far as I can tell, they always include an
attempt to cd to the project root and then run the maintainer target.
Signed-off-by: Eli Schwartz gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1359
Signed-off-by: Sam James gentoo.org>
lib/portage/package/ebuild/doebuild.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index c7fa1d1e06..403836b80b 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -2383,7 +2383,7 @@ def _check_build_log(mysettings, out=None):
# Configuration:
# Automake: ${SHELL}
/var/tmp/portage/dev-libs/yaz-3.0.47/work/yaz-3.0.47/config/missing --run
automake-1.10
am_maintainer_mode_re = re.compile(
-r"/missing( --run|'|) (automake|autoconf|autoheader|aclocal)"
+r"\bcd\b.*&&.*/missing( --run|'|)
(automake|autoconf|autoheader|aclocal)"
)
am_maintainer_mode_exclude_re = re.compile(r"^\s*Automake:\s")
[gentoo-commits] proj/portage:master commit in: lib/portage/repository/, /
commit: ea04a583fab9739dbe05e8d8381ed2803e7dce88 Author: James Le Cuirot gentoo org> AuthorDate: Tue Jul 9 13:07:58 2024 + Commit: James Le Cuirot gentoo org> CommitDate: Wed Jul 10 16:55:20 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ea04a583 config: Allow a repository to be configured using one of its aliases Currently, the `[name]` in repos.conf can only match the primary name. This is inconvenient if a repository wants to change its name without breaking existing configurations. This raises the question of whether to then use the primary name or the alias in the UI and in the vardb. I went with the primary name because this is presumably the name that the repository actually wants you to use. If the configured name matches neither the primary name nor an alias, then emaint sync will simply claim that it is not found, but that behaviour is unchanged from before. Bug: https://bugs.gentoo.org/935830 Closes: https://github.com/gentoo/portage/pull/1358 Signed-off-by: James Le Cuirot gentoo.org> NEWS | 3 +++ lib/portage/repository/config.py | 10 ++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index d5a03533d9..53ba9f5fb1 100644 --- a/NEWS +++ b/NEWS @@ -22,6 +22,9 @@ Bug fixes: * vartree, movefile: Warn when rewriting a symlink (bug #934514). +* repository: config: Allow a repository to be configured using one of its + aliases rather than its primary name (bug #935830). + portage-3.0.65 (2024-06-04) -- diff --git a/lib/portage/repository/config.py b/lib/portage/repository/config.py index c9dfffa22c..46a2d83856 100644 --- a/lib/portage/repository/config.py +++ b/lib/portage/repository/config.py @@ -943,7 +943,9 @@ class RepoConfigLoader: del prepos[repo_name] continue -if repo.name != repo_name: +if repo.name != repo_name and ( +repo.aliases is None or repo_name not in repo.aliases +): writemsg_level( "!!! %s\n" % _( @@ -957,13 +959,13 @@ class RepoConfigLoader: del prepos[repo_name] continue -location_map[repo.location] = repo_name -treemap[repo_name] = repo.location +location_map[repo.location] = repo.name +treemap[repo.name] = repo.location # Add alias mappings, but never replace unaliased mappings. for repo_name, repo in list(prepos.items()): names = set() -names.add(repo_name) +names.add(repo.name) if repo.aliases: aliases = stack_lists([repo.aliases], incremental=True) names.update(aliases)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, /, lib/portage/util/
commit: 4d17764863896903df4d18fe7c6b45635a18f6c4
Author: Ulrich Müller gentoo org>
AuthorDate: Wed Jun 19 08:08:04 2024 +
Commit: Ulrich Müller gentoo org>
CommitDate: Wed Jun 19 18:17:06 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4d177648
vartree, movefile: Warn when rewriting a symlink
See PMS section 13.4.1 (Rewriting):
Any absolute symlink whose link starts with D must be rewritten with
the leading D removed. The package manager should issue a notice when
doing this.
Bug: https://bugs.gentoo.org/934514
Signed-off-by: Ulrich Müller gentoo.org>
NEWS | 2 ++
lib/portage/dbapi/vartree.py | 10 ++
lib/portage/util/movefile.py | 5 +
3 files changed, 17 insertions(+)
diff --git a/NEWS b/NEWS
index b4b378e7a0..d5a03533d9 100644
--- a/NEWS
+++ b/NEWS
@@ -20,6 +20,8 @@ Bug fixes:
working ebuilds. Future EAPIs will need to adjust the logic
added by this change. See bug #907061.
+* vartree, movefile: Warn when rewriting a symlink (bug #934514).
+
portage-3.0.65 (2024-06-04)
--
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index beb1a6486f..0c41d408c3 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -5563,6 +5563,16 @@ class dblink:
myabsto = myabsto.lstrip(sep)
if self.settings and self.settings["D"]:
if myto.startswith(self.settings["D"]):
+self._eqawarn(
+"preinst",
+[
+_(
+"QA Notice: Absolute symlink %s points to
%s inside the image directory.\n"
+"Removing the leading %s from its path."
+)
+% (mydest, myto, self.settings["D"])
+],
+)
myto = myto[len(self.settings["D"]) - 1 :]
# myrealto contains the path of the real file to which this
symlink points.
# we can simply test for existence of this file to see if the
target has been merged yet
diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
index 75100a3acd..7b880d2e3e 100644
--- a/lib/portage/util/movefile.py
+++ b/lib/portage/util/movefile.py
@@ -210,6 +210,11 @@ def movefile(
try:
target = os.readlink(src)
if mysettings and "D" in mysettings and
target.startswith(mysettings["D"]):
+writemsg(
+f"!!! {_('Absolute symlink points to image
directory.')}\n",
+noiselevel=-1,
+)
+writemsg(f"!!! {dest} -> {target}\n", noiselevel=-1)
target = target[len(mysettings["D"]) - 1 :]
# Atomically update the path if it exists.
try:
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/
commit: 819c86325ce6b966d56d5f4848cf762767ab2bf9 Author: Sam James gentoo org> AuthorDate: Mon Jun 17 00:24:52 2024 + Commit: Sam James gentoo org> CommitDate: Mon Jun 17 00:24:52 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=819c8632 ebuild: fix typo in comment Signed-off-by: Sam James gentoo.org> lib/portage/package/ebuild/doebuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py index 21ff5a77fd..c7fa1d1e06 100644 --- a/lib/portage/package/ebuild/doebuild.py +++ b/lib/portage/package/ebuild/doebuild.py @@ -2390,7 +2390,7 @@ def _check_build_log(mysettings, out=None): make_jobserver_re = re.compile(r"g?make\[\d+\]: warning: jobserver unavailable:") make_jobserver = [] -# we deduplicate these since they is repeated for every setup.py call +# we deduplicate these since they are repeated for every setup.py call setuptools_warn = set() setuptools_warn_re = re.compile(r".*\/setuptools\/.*: .*Warning: (.*)") # skip useless version normalization warnings
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/
commit: 11c51d7c78f48d4c9842e0d475c26b7068f18c3e
Author: Eli Schwartz gmail com>
AuthorDate: Mon Jun 17 00:12:29 2024 +
Commit: Sam James gentoo org>
CommitDate: Mon Jun 17 00:20:41 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=11c51d7c
ebuild: fix maintainer mode checks to work with modern autotools
Modern autotools does not use the --run argument to "missing", so the
check essentially never ever ever ever fired anywhere.
The GNU "missing" script is actually allowed to be run by any software
at all, so checking for "missing --run" was always wrong. Indeed, there
are some packages which use it for their own purposes and added
suppressions for this FP.
The correct solution really is to check for *maintainer mode* by
checking whether *maintainer* programs are run (via "missing"). This
also means we get to check for specific programs which autotools.eclass
would be capable of handling, and don't need to arbitrarily exclude
stuff like help2man (???) which makes things somewhat simpler.
It should be noted that I have observed 3 scenarios for the missing
script to be run via:
- the missing script is surrounded by single quotes, followed by the
unquoted command
- the missing script is unquoted, and is followed by the unquoted
command
- legacy: the missing script is unquoted and is followed by --run
We have to handle all three cases via a regex group.
Signed-off-by: Eli Schwartz gmail.com>
Signed-off-by: Sam James gentoo.org>
lib/portage/package/ebuild/doebuild.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 6641cd8341..21ff5a77fd 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -2382,10 +2382,10 @@ def _check_build_log(mysettings, out=None):
#
# Configuration:
# Automake: ${SHELL}
/var/tmp/portage/dev-libs/yaz-3.0.47/work/yaz-3.0.47/config/missing --run
automake-1.10
-am_maintainer_mode_re = re.compile(r"/missing --run ")
-am_maintainer_mode_exclude_re = re.compile(
-r"(/missing --run
(autoheader|autotest|help2man|makeinfo)|^\s*Automake:\s)"
+am_maintainer_mode_re = re.compile(
+r"/missing( --run|'|) (automake|autoconf|autoheader|aclocal)"
)
+am_maintainer_mode_exclude_re = re.compile(r"^\s*Automake:\s")
make_jobserver_re = re.compile(r"g?make\[\d+\]: warning: jobserver
unavailable:")
make_jobserver = []
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: f620a0769a509966295954c2b0c76e46e8fb4289
Author: Zac Medico gentoo org>
AuthorDate: Sun Jun 2 21:53:04 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Jun 9 17:53:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f620a076
tar_safe_extract: Use tarfile.fully_trusted_filter
This suppresses a DeprecationWarning triggered because the
tarfile.data_filter will become the new default in python3.14.
The fully_trusted filter should be suitable here because
tar_safe_extract already performs security validation on
tar members prior to extraction.
Bug: https://bugs.gentoo.org/933433
Signed-off-by: Zac Medico gentoo.org>
lib/portage/gpkg.py | 9 +
1 file changed, 9 insertions(+)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 9606f6d3c8..fdb54c69b8 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -628,6 +628,15 @@ class tar_safe_extract:
if self.closed:
raise OSError("Tar file is closed.")
temp_dir = tempfile.TemporaryDirectory(dir=dest_dir)
+# The below tar member security checks can be refactored as a filter
function
+# that raises an exception. Use tarfile.fully_trusted_filter for now,
which
+# is simply an identity function:
+# def fully_trusted_filter(member, dest_path):
+# return member
+try:
+self.tar.extraction_filter = tarfile.fully_trusted_filter
+except AttributeError:
+pass
try:
while True:
member = self.tar.next()
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/, lib/portage/sync/modules/
commit: 8bb7aecf7e5c922911192d0df63853c5c75d9f8a
Author: Alexey Gladkov kernel org>
AuthorDate: Tue Jun 4 15:31:06 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Jun 4 15:40:25 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8bb7aecf
sync/zipfile: Install zipfile sync method
Add files that were accidentally forgotten when adding zipfile sync
method.
Fixes: 80445d9b0 ("sync: Add method to download zip archives")
Signed-off-by: Alexey Gladkov kernel.org>
Closes: https://github.com/gentoo/portage/pull/1340
Signed-off-by: Zac Medico gentoo.org>
lib/portage/sync/modules/meson.build | 1 +
lib/portage/sync/modules/zipfile/meson.build | 8
2 files changed, 9 insertions(+)
diff --git a/lib/portage/sync/modules/meson.build
b/lib/portage/sync/modules/meson.build
index fab2878e92..ba0b6f278b 100644
--- a/lib/portage/sync/modules/meson.build
+++ b/lib/portage/sync/modules/meson.build
@@ -12,3 +12,4 @@ subdir('mercurial')
subdir('rsync')
subdir('svn')
subdir('webrsync')
+subdir('zipfile')
diff --git a/lib/portage/sync/modules/zipfile/meson.build
b/lib/portage/sync/modules/zipfile/meson.build
new file mode 100644
index 00..46006aea7e
--- /dev/null
+++ b/lib/portage/sync/modules/zipfile/meson.build
@@ -0,0 +1,8 @@
+py.install_sources(
+[
+'zipfile.py',
+'__init__.py',
+],
+subdir : 'portage/sync/modules/zipfile',
+pure : not native_extensions
+)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/_emerge/
commit: 3a9f2c09eb75f47cf3ae15fa4ebe671548a66870
Author: Zac Medico gentoo org>
AuthorDate: Mon Jun 3 01:18:44 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jun 3 01:55:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3a9f2c09
Add binpkgs info to mtimedb resume data
In order to fix emerge --resume and --keep-going to make
appropriate binary package selections, store a list of
binpkgs in the resume data. By adding the data as a new
key which older versions of portage will ignore, the
extension is backward compatible.
Without this fix, emerge --resume and --keep-going make
poor package selection choices which do not account for
the --binpkg-respect-use option.
Bug: https://bugs.gentoo.org/933442
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/Scheduler.py | 13 +
lib/_emerge/actions.py | 7 +++
lib/_emerge/depgraph.py| 21 +
lib/portage/tests/util/test_mtimedb.py | 30 +-
4 files changed, 66 insertions(+), 5 deletions(-)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index d913cd2dc6..614df9e783 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -2116,6 +2116,19 @@ class Scheduler(PollScheduler):
for x in self._mergelist
if isinstance(x, Package) and x.operation == "merge"
]
+# Store binpkgs using the same keys as $PKGDIR/Packages plus EROOT.
+mtimedb["resume"]["binpkgs"] = [
+{
+"CPV": str(x.cpv),
+"BUILD_ID": x.cpv.build_id,
+"BUILD_TIME": x.cpv.build_time,
+"MTIME": x.cpv.mtime,
+"SIZE": x.cpv.file_size,
+"EROOT": x.root,
+}
+for x in self._mergelist
+if isinstance(x, Package) and x.type_name == "binary"
+]
mtimedb.commit()
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 512e470ad6..43d936fd14 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -216,6 +216,13 @@ def action_build(
if not isinstance(favorites, list):
del mtimedb[k]
continue
+binpkgs = resume_data.get("binpkgs")
+if binpkgs and (
+not isinstance(binpkgs, list)
+or any(not isinstance(x, dict) for x in binpkgs)
+):
+del mtimedb[k]
+continue
resume = False
if "--resume" in myopts and ("resume" in mtimedb or "resume_backup" in
mtimedb):
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 3adc04bcfb..a05404d9c2 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -11066,6 +11066,13 @@ class depgraph:
else:
args = []
+binpkgs_map = {}
+binpkgs = resume_data.get("binpkgs")
+if binpkgs:
+for x in binpkgs:
+if isinstance(x, dict) and "EROOT" in x and "CPV" in x:
+binpkgs_map[(x["EROOT"], x["CPV"])] = x
+
serialized_tasks = []
masked_tasks = []
for x in mergelist:
@@ -11096,8 +11103,22 @@ class depgraph:
except InvalidAtom:
continue
+if pkg_type == "binary":
+binpkg_info = binpkgs_map.get((myroot, pkg_key))
+else:
+binpkg_info = False
+
pkg = None
for pkg in self._iter_match_pkgs(root_config, pkg_type, atom):
+if binpkg_info:
+if not (
+pkg.cpv.build_id == binpkg_info.get("BUILD_ID")
+and pkg.cpv.build_time == binpkg_info.get("BUILD_TIME")
+and pkg.cpv.mtime == binpkg_info.get("MTIME")
+and pkg.cpv.file_size == binpkg_info.get("SIZE")
+):
+continue
+
if not self._pkg_visibility_check(
pkg
) or self._frozen_config.excluded_pkgs.findAtomForPackage(
diff --git a/lib/portage/tests/util/test_mtimedb.py
b/lib/portage/tests/util/test_mtimedb.py
index d80b4f1daf..6e3c2bee76 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Gentoo Foundation
+# Copyright 2022-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from unittest.mock import patch, mock_open
@@ -30,6 +30,16 @@ _ONE_RESUME_LIST_JSON = b"""{
"/usr/local/lib64": 1711784303
},
"resume": {
+ "binpkgs": [
+ {
+ "CPV": "another-cat/another-package-4.3.2-r1",
+ "BUILD_ID": 1,
+ "BUILD_TIME": 1710959527,
+ "MTIME": 1710966082,
+
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: eb855b8cd1248f49649003dcfb9bf009b70e88cb
Author: Zac Medico gentoo org>
AuthorDate: Sun Jun 2 17:56:44 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Jun 2 18:05:35 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=eb855b8c
tar_stream_writer: Add missing error attribute
This attribute was previously initialized only
in an exception handler.
Fixes: b8c3f38ec5ee ("Add more error handling for binpkgs")
Bug: https://bugs.gentoo.org/933385
Signed-off-by: Zac Medico gentoo.org>
lib/portage/gpkg.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index fc4d7b1fb6..9606f6d3c8 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -100,6 +100,7 @@ class tar_stream_writer:
self.closed = False
self.container = container
self.killed = False
+self.error = False
self.tar_format = tar_format
self.tarinfo = tarinfo
self.uid = uid
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 1a7fc63d20ad2e1292be3697c105c2d7e1691f91
Author: Zac Medico gentoo org>
AuthorDate: Sat Jun 1 04:56:45 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Jun 1 19:18:35 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1a7fc63d
MergeProcess: Pass bintree to subprocess
It's required for FEATURES=*-backup.
Fixes: b9a85ff987ea ("MergeProcess: Support QueryCommand with spawn start
method")
Bug: https://bugs.gentoo.org/933297
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/_MergeProcess.py
b/lib/portage/dbapi/_MergeProcess.py
index d9ab2b47aa..34e39eb229 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2023 Gentoo Authors
+# Copyright 2010-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -183,11 +183,12 @@ class MergeProcess(ForkProcess):
# Since the entire QueryCommand._db is not required, only pass
# in tree types that QueryCommand specifically requires.
+# NOTE: For FEATURES=*-backup bintree is needed (bug 933297).
child_db = {}
parent_db = portage.db if QueryCommand._db is None else
QueryCommand._db
for root in parent_db:
child_db[root] = {}
-for tree_type in ("vartree", "porttree"):
+for tree_type in ("bintree", "porttree", "vartree"):
child_db[root][tree_type] = parent_db[root][tree_type]
self.target = functools.partial(
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/
commit: 120b2ec988eebf6cd90365d5b50a1a718eebb116
Author: Zac Medico gentoo org>
AuthorDate: Tue May 28 05:37:13 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue May 28 05:37:13 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=120b2ec9
atomic_ofstream: fix follow_symlinks fallback and default file mode
Handle OSError from mkstemp for (default) follow_symlinks mode,
not following the symlink if necessary (the target's parent may
not exist or may be readonly). This restores the fallback
behavior that existed before the introduction of mkstemp in
commit de19f3a7215d.
Handle missing _file and _tmp_name attributes during close.
Also set the default file mode respecting umask if a previous
file does not exist, which fixes the mode of CONTENTS files
since mkstemp.
Fixes: de19f3a7215d ("atomic_ofstream: Use mkstemp rather than getpid (pid
namespace safety)")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/meson.build | 1 +
lib/portage/tests/util/test_atomic_ofstream.py | 85 ++
lib/portage/util/__init__.py | 45 +-
3 files changed, 116 insertions(+), 15 deletions(-)
diff --git a/lib/portage/tests/util/meson.build
b/lib/portage/tests/util/meson.build
index 010dfa7849..7f4db871f4 100644
--- a/lib/portage/tests/util/meson.build
+++ b/lib/portage/tests/util/meson.build
@@ -1,5 +1,6 @@
py.install_sources(
[
+'test_atomic_ofstream.py',
'test_checksum.py',
'test_digraph.py',
'test_file_copier.py',
diff --git a/lib/portage/tests/util/test_atomic_ofstream.py
b/lib/portage/tests/util/test_atomic_ofstream.py
new file mode 100644
index 00..bbaf0f1b06
--- /dev/null
+++ b/lib/portage/tests/util/test_atomic_ofstream.py
@@ -0,0 +1,85 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import os
+import stat
+import tempfile
+
+from portage.tests import TestCase
+from portage.util import atomic_ofstream
+
+
+class AtomicOFStreamTestCase(TestCase):
+def test_enospc_rollback(self):
+file_name = "foo"
+start_dir = os.getcwd()
+with tempfile.TemporaryDirectory() as tempdir:
+try:
+os.chdir(tempdir)
+with self.assertRaises(OSError):
+with atomic_ofstream(file_name) as f:
+f.write("hello")
+raise OSError(errno.ENOSPC, "No space left on device")
+self.assertFalse(os.path.exists(file_name))
+self.assertEqual(os.listdir(tempdir), [])
+finally:
+os.chdir(start_dir)
+
+def test_open_failure(self):
+file_name = "bad/path"
+start_dir = os.getcwd()
+with tempfile.TemporaryDirectory() as tempdir:
+try:
+os.chdir(tempdir)
+with self.assertRaises(OSError):
+with atomic_ofstream(file_name):
+pass
+self.assertEqual(os.listdir(tempdir), [])
+finally:
+os.chdir(start_dir)
+
+def test_broken_symlink(self):
+content = "foo"
+broken_symlink = "symlink"
+symlink_targets = (("foo/bar/baz", False), ("baz", True))
+start_dir = os.getcwd()
+for symlink_target, can_follow in symlink_targets:
+with tempfile.TemporaryDirectory() as tempdir:
+try:
+os.chdir(tempdir)
+with open(broken_symlink, "w") as f:
+default_file_mode =
stat.S_IMODE(os.fstat(f.fileno()).st_mode)
+os.unlink(broken_symlink)
+os.symlink(symlink_target, broken_symlink)
+with atomic_ofstream(broken_symlink) as f:
+f.write(content)
+with open(broken_symlink) as f:
+self.assertEqual(f.read(), content)
+self.assertEqual(os.path.islink(broken_symlink),
can_follow)
+self.assertEqual(
+stat.S_IMODE(os.stat(broken_symlink).st_mode),
default_file_mode
+)
+finally:
+os.chdir(start_dir)
+
+def test_preserves_mode(self):
+file_name = "foo"
+file_mode = 0o604
+start_dir = os.getcwd()
+with tempfile.TemporaryDirectory() as tempdir:
+try:
+os.chdir(tempdir)
+with open(file_name, "wb"):
+pass
+self.assertNotEqual(stat.S_IMODE(os.stat(file_name).st_mode),
file_mode)
+os.chmod(file_name, file_mode)
+st_before = os.stat(file_name)
+self.assertEqual(stat.S_IMODE(st_before.st_mode), file_mode)
+with atomic_ofstream(file_name):
+
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: de19f3a7215d64d22dcc0f779314de1f1199963f Author: Zac Medico gentoo org> AuthorDate: Mon May 27 18:47:30 2024 + Commit: Zac Medico gentoo org> CommitDate: Mon May 27 20:01:49 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=de19f3a7 atomic_ofstream: Use mkstemp rather than getpid (pid namespace safety) Bug: https://bugs.gentoo.org/851015 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/__init__.py | 47 ++-- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py index 0c88068dda..f338f274aa 100644 --- a/lib/portage/util/__init__.py +++ b/lib/portage/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2004-2023 Gentoo Authors +# Copyright 2004-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from portage.cache.mappings import UserDict @@ -70,6 +70,7 @@ import shlex import stat import string import sys +import tempfile import traceback import glob from typing import Optional, TextIO @@ -1446,21 +1447,22 @@ class atomic_ofstream(AbstractContextManager, ObjectProxy): if follow_links: canonical_path = os.path.realpath(filename) object.__setattr__(self, "_real_name", canonical_path) -tmp_name = "%s.%i" % (canonical_path, portage.getpid()) +parent, basename = os.path.split(canonical_path) +fd, tmp_name = tempfile.mkstemp(prefix=basename, dir=parent) +object.__setattr__(self, "_tmp_name", tmp_name) try: object.__setattr__( self, "_file", open_func( -_unicode_encode( -tmp_name, encoding=_encodings["fs"], errors="strict" -), +fd, mode=mode, **kargs, ), ) return -except OSError as e: +except OSError: +os.close(fd) if canonical_path == filename: raise # Ignore this error, since it's irrelevant @@ -1468,16 +1470,22 @@ class atomic_ofstream(AbstractContextManager, ObjectProxy): # new error if necessary. object.__setattr__(self, "_real_name", filename) -tmp_name = "%s.%i" % (filename, portage.getpid()) -object.__setattr__( -self, -"_file", -open_func( -_unicode_encode(tmp_name, encoding=_encodings["fs"], errors="strict"), -mode=mode, -**kargs, -), -) +parent, basename = os.path.split(filename) +fd, tmp_name = tempfile.mkstemp(prefix=basename, dir=parent) +object.__setattr__(self, "_tmp_name", tmp_name) +try: +object.__setattr__( +self, +"_file", +open_func( +fd, +mode=mode, +**kargs, +), +) +except OSError: +os.close(fd) +raise def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: @@ -1498,13 +1506,14 @@ class atomic_ofstream(AbstractContextManager, ObjectProxy): and performs the atomic replacement via os.rename(). If the abort() method has been called, then the temp file is closed and removed.""" f = object.__getattribute__(self, "_file") +tmp_name = object.__getattribute__(self, "_tmp_name") real_name = object.__getattribute__(self, "_real_name") if not f.closed: try: f.close() if not object.__getattribute__(self, "_aborted"): try: -apply_stat_permissions(f.name, os.stat(real_name)) +apply_stat_permissions(tmp_name, os.stat(real_name)) except OperationNotPermitted: pass except FileNotFound: @@ -1514,12 +1523,12 @@ class atomic_ofstream(AbstractContextManager, ObjectProxy): pass else: raise -os.rename(f.name, real_name) +os.rename(tmp_name, real_name) finally: # Make sure we cleanup the temp file # even if an exception is raised. try: -os.unlink(f.name) +os.unlink(tmp_name) except OSError as oe: pass
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 2db89a16ab87b85004216959ec6bc508575d96a0
Author: Zac Medico gentoo org>
AuthorDate: Mon May 27 18:03:32 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon May 27 18:04:41 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2db89a16
binarytree: Rewrite remote index only on change
I noticed that the remote index was rewritten with a
new DOWNLOAD_TIMESTAMP even while frozen, and this
patch fixed it.
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dbapi/bintree.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 64dfee4faa..b32dea1eae 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1400,6 +1400,7 @@ class binarytree:
except OSError as e:
if e.errno != errno.ENOENT:
raise
+changed = True
local_timestamp = pkgindex.header.get("TIMESTAMP", None)
try:
download_timestamp =
float(pkgindex.header.get("DOWNLOAD_TIMESTAMP", 0))
@@ -1574,6 +1575,7 @@ class binarytree:
noiselevel=-1,
)
except UseCachedCopyOfRemoteIndex:
+changed = False
desc = "frozen" if repo.frozen else "up-to-date"
writemsg_stdout("\n")
writemsg_stdout(
@@ -1611,7 +1613,7 @@ class binarytree:
os.unlink(tmp_filename)
except OSError:
pass
-if pkgindex is rmt_idx:
+if pkgindex is rmt_idx and changed:
pkgindex.modified = False # don't update the header
pkgindex.header["DOWNLOAD_TIMESTAMP"] = "%d" % time.time()
try:
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/rsync/
commit: a671334b7c7b64bc779f1c2bfb4ed659d97c0d19
Author: Pavel Balaev void so>
AuthorDate: Mon Mar 18 11:29:34 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun May 26 23:27:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a671334b
sync: don't use ipv6 for rsync when it's disabled
socket.has_ipv6 gives a false result:
$ sysctl net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.all.disable_ipv6 = 1
$ python
Python 3.11.8 (main, Feb 24 2024, 17:10:38) [GCC 13.2.1 20240210] on linux
>>> import socket
>>> socket.has_ipv6
True
This patch uses the portage.process.has_ipv6() function,
which returns the correct result.
Bug: https://bugs.gentoo.org/927241
Signed-off-by: Pavel Balaev void.so>
Closes: https://github.com/gentoo/portage/pull/1309
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/rsync/rsync.py | 7 +++
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/lib/portage/sync/modules/rsync/rsync.py
b/lib/portage/sync/modules/rsync/rsync.py
index 15c3eb4da5..e89221ebc9 100644
--- a/lib/portage/sync/modules/rsync/rsync.py
+++ b/lib/portage/sync/modules/rsync/rsync.py
@@ -19,6 +19,7 @@ from portage import _unicode_decode
from portage import os
from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
from portage.output import create_color_func, yellow, blue, bold
+from portage.process import has_ipv6
from portage.sync.getaddrinfo_validate import getaddrinfo_validate
from portage.sync.syncbase import NewBase
from portage.util import writemsg, writemsg_level, writemsg_stdout
@@ -252,9 +253,7 @@ class RsyncSync(NewBase):
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
-elif socket.has_ipv6 and (
-"-6" in all_rsync_opts or "--ipv6" in all_rsync_opts
-):
+elif has_ipv6() and ("-6" in all_rsync_opts or "--ipv6" in
all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
@@ -278,7 +277,7 @@ class RsyncSync(NewBase):
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
-if socket.has_ipv6:
+if has_ipv6():
AF_INET6 = socket.AF_INET6
ips_v4 = []
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/, lib/portage/
commit: d00ecdce359c8bc3d3520aef58d5808c7d15020b
Author: Pavel Balaev void so>
AuthorDate: Wed Mar 20 14:35:54 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun May 26 23:27:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d00ecdce
process: make has_ipv6 a public function
Signed-off-by: Pavel Balaev void.so>
Signed-off-by: Sam James gentoo.org>
lib/portage/process.py| 8
lib/portage/tests/process/test_unshare_net.py | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 1bc0c507c2..6e4e0d7162 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -631,8 +631,8 @@ def spawn(
fd_pipes[1] = pw
fd_pipes[2] = pw
-# Cache _has_ipv6() result for use in child processes.
-_has_ipv6()
+# Cache has_ipv6() result for use in child processes.
+has_ipv6()
# This caches the libc library lookup and _unshare_validator results
# in the current process, so that results are cached for use in
@@ -751,7 +751,7 @@ def spawn(
__has_ipv6 = None
-def _has_ipv6():
+def has_ipv6():
"""
Test that both userland and kernel support IPv6, by attempting
to create a socket and listen on any unused port of the IPv6
@@ -812,7 +812,7 @@ def _configure_loopback_interface():
ifindex = rtnl.get_link_ifindex(b"lo")
rtnl.set_link_up(ifindex)
rtnl.add_address(ifindex, socket.AF_INET, "10.0.0.1", 8)
-if _has_ipv6():
+if has_ipv6():
rtnl.add_address(ifindex, socket.AF_INET6, "fd::1", 8)
except OSError as e:
writemsg(
diff --git a/lib/portage/tests/process/test_unshare_net.py
b/lib/portage/tests/process/test_unshare_net.py
index ad3b288ef4..56d96da05f 100644
--- a/lib/portage/tests/process/test_unshare_net.py
+++ b/lib/portage/tests/process/test_unshare_net.py
@@ -42,7 +42,7 @@ class UnshareNetTestCase(TestCase):
f"Unable to unshare:
{errno.errorcode.get(self.ABILITY_TO_UNSHARE, '?')}"
)
env = os.environ.copy()
-env["IPV6"] = "1" if portage.process._has_ipv6() else ""
+env["IPV6"] = "1" if portage.process.has_ipv6() else ""
self.assertEqual(
portage.process.spawn(
[BASH_BINARY, "-c", UNSHARE_NET_TEST_SCRIPT],
unshare_net=True, env=env
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/resolver/
commit: ff22b5bf954574f1fad789fc148945d67c3a1215 Author: Zac Medico gentoo org> AuthorDate: Sun May 26 18:57:38 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun May 26 18:57:58 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ff22b5bf test_tar_merge_order.py: Fix unused pytest unused-import Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/resolver/test_tar_merge_order.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/portage/tests/resolver/test_tar_merge_order.py b/lib/portage/tests/resolver/test_tar_merge_order.py index 4bd9b4df4a..c66d01ee31 100644 --- a/lib/portage/tests/resolver/test_tar_merge_order.py +++ b/lib/portage/tests/resolver/test_tar_merge_order.py @@ -2,7 +2,6 @@ # Distributed under the terms of the GNU General Public License v2 import os -import pytest from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import (
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/resolver/
commit: 11bded10a2a4f143e8d7f8ccb2f91f2b6fed59b5
Author: Sam James gentoo org>
AuthorDate: Sun May 26 14:44:56 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun May 26 18:27:02 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=11bded10
tests: add testcase for app-arch/tar merge order with binpkgs
In the bug, dilfridge reports releng@ is hitting the following:
```
$ emerge -epvk world | grep -E "(app-arch/tar|sys-apps/acl)"
[ebuild N ] sys-apps/acl-2.3.2-r1::gentoo USE="nls -static-libs" 363 KiB
[binary R] app-arch/tar-1.35-1::gentoo USE="acl* nls* xattr* -minimal
(-selinux) -verify-sig" 0 KiB
[...]
```
Test for bug #922629 where binary app-arch/tar[acl] was merged
before its dependency sys-apps/acl (with virtual/acl merged but
unsatisfied).
It (appears to be) a bad interaction with @system containing
app-alternatives/tar
plus a circular dependency on app-arch/tar. The USE change is also important,
as e.g. dropping "sys-apps/attr nls" from package.use makes things okay.
XFAIL'd for now.
Bug: https://bugs.gentoo.org/922629
Signed-off-by: Sam James gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1332
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/resolver/meson.build | 1 +
lib/portage/tests/resolver/test_tar_merge_order.py | 495 +
2 files changed, 496 insertions(+)
diff --git a/lib/portage/tests/resolver/meson.build
b/lib/portage/tests/resolver/meson.build
index 8892c78131..ea948982e7 100644
--- a/lib/portage/tests/resolver/meson.build
+++ b/lib/portage/tests/resolver/meson.build
@@ -81,6 +81,7 @@ py.install_sources(
'test_slot_operator_update_probe_parent_downgrade.py',
'test_solve_non_slot_operator_slot_conflicts.py',
'test_targetroot.py',
+'test_tar_merge_order.py',
'test_unmerge_order.py',
'test_unnecessary_slot_upgrade.py',
'test_update.py',
diff --git a/lib/portage/tests/resolver/test_tar_merge_order.py
b/lib/portage/tests/resolver/test_tar_merge_order.py
new file mode 100644
index 00..7e1a18bc21
--- /dev/null
+++ b/lib/portage/tests/resolver/test_tar_merge_order.py
@@ -0,0 +1,495 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import pytest
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+ResolverPlayground,
+ResolverPlaygroundTestCase,
+)
+
+
+class TarMergeOrderTestCase(TestCase):
+@pytest.mark.xfail(reason="bug #922629 isn't yet fixed")
+def testTarMergeOrder(self):
+"""
+Test for bug #922629 where binary app-arch/tar[acl] was merged
+before its dependency sys-apps/acl (with virtual/acl merged but
+unsatisfied).
+
+It poorly interacted with @system containing app-alternatives/tar
+as a circular dependency on app-arch/tar.
+"""
+
+ebuilds = {
+"app-alternatives/tar-0-1": {
+"EAPI": "8",
+"RDEPEND": """
+!=app-arch/tar-1.34-r2 )
+libarchive? ( app-arch/libarchive )
+""",
+"IUSE": "+gnu libarchive",
+"REQUIRED_USE": "^^ ( gnu libarchive )",
+},
+"app-arch/libarchive-3.7.4": {"EAPI": "8"},
+"app-arch/tar-1.35": {
+"EAPI": "8",
+"RDEPEND": """
+acl? ( virtual/acl )
+""",
+"DEPEND": """
+acl? ( virtual/acl )
+xattr? ( sys-apps/attr )
+""",
+"BDEPEND": """
+nls? ( sys-devel/gettext )
+""",
+"IUSE": "acl nls xattr",
+},
+"virtual/acl-0-r2": {
+"EAPI": "8",
+"RDEPEND": ">=sys-apps/acl-2.2.52-r1",
+},
+"sys-devel/gettext-0.22.4": {
+"EAPI": "8",
+"RDEPEND": """
+acl? ( virtual/acl )
+xattr? ( sys-apps/attr )
+""",
+"DEPEND": """
+acl? ( virtual/acl )
+xattr? ( sys-apps/attr )
+""",
+"IUSE": "acl nls xattr",
+},
+"sys-apps/attr-2.5.2-r1": {
+"EAPI": "8",
+"BDEPEND": "nls? ( sys-devel/gettext )",
+"IUSE": "nls",
+},
+"sys-apps/acl-2.3.2-r1": {
+"EAPI": "8",
+"DEPEND": ">=sys-apps/attr-2.4.47-r1",
+"RDEPEND": ">=sys-apps/attr-2.4.47-r1",
+"BDEPEND": "nls? ( sys-devel/gettext )",
+"IUSE": "nls",
+},
+}
+
+installed = {
+"app-alternatives/tar-0-1": {
+"EAPI": "8",
+
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/sync/, lib/portage/dbapi/
commit: 5aed7289d516fab5b63557da46348125eabab368
Author: Zac Medico gentoo org>
AuthorDate: Thu Mar 14 04:09:34 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat May 25 22:08:15 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5aed7289
bintree: Add REPO_REVISIONS to package index header
As a means for binhost clients to select source repo
revisions which are consistent with binhosts, inject
REPO_REVISIONS from a package into the index header,
using a history of synced revisions to guarantee
forward progress. This queries the relevant repos to
check if any new revisions have appeared in the
absence of a proper sync operation.
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dbapi/bintree.py | 67 -
lib/portage/tests/sync/test_sync_local.py | 71 +--
2 files changed, 124 insertions(+), 14 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 221afbd154..64dfee4faa 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -48,6 +48,7 @@ from portage.exception import (
from portage.localization import _
from portage.output import colorize
from portage.package.ebuild.profile_iuse import iter_iuse_vars
+from portage.sync.revision_history import get_repo_revision_history
from portage.util import ensure_dirs
from portage.util.file_copy import copyfile
from portage.util.futures import asyncio
@@ -62,6 +63,7 @@ from portage import _unicode_encode
import codecs
import errno
import io
+import json
import re
import shlex
import stat
@@ -135,13 +137,19 @@ class bindbapi(fakedbapi):
"USE",
"_mtime_",
}
+# Keys required only when initially adding a package.
+self._init_aux_keys = {
+"REPO_REVISIONS",
+}
self._aux_cache = {}
self._aux_cache_slot_dict_cache = None
@property
def _aux_cache_slot_dict(self):
if self._aux_cache_slot_dict_cache is None:
-self._aux_cache_slot_dict_cache =
slot_dict_class(self._aux_cache_keys)
+self._aux_cache_slot_dict_cache = slot_dict_class(
+chain(self._aux_cache_keys, self._init_aux_keys)
+)
return self._aux_cache_slot_dict_cache
def __getstate__(self):
@@ -1791,6 +1799,11 @@ class binarytree:
pkgindex = self._new_pkgindex()
d = self._inject_file(pkgindex, cpv, full_path)
+repo_revisions = (
+json.loads(d["REPO_REVISIONS"]) if d.get("REPO_REVISIONS")
else None
+)
+if repo_revisions:
+self._inject_repo_revisions(pkgindex.header, repo_revisions)
self._update_pkgindex_header(pkgindex.header)
self._pkgindex_write(pkgindex)
@@ -1872,7 +1885,7 @@ class binarytree:
@return: package metadata
"""
if keys is None:
-keys = self.dbapi._aux_cache_keys
+keys = chain(self.dbapi._aux_cache_keys, self.dbapi._init_aux_keys)
metadata = self.dbapi._aux_cache_slot_dict()
else:
metadata = {}
@@ -1916,6 +1929,56 @@ class binarytree:
return metadata
+def _inject_repo_revisions(self, header, repo_revisions):
+"""
+Inject REPO_REVISIONS from a package into the index header,
+using a history of synced revisions to guarantee forward
+progress. This queries the relevant repos to check if any
+new revisions have appeared in the absence of a proper sync
+operation.
+
+This does not expose REPO_REVISIONS that do not appear in
+the sync history, since such revisions suggest that the
+package was not built locally, and in this case its
+REPO_REVISIONS are not intended to be exposed.
+"""
+synced_repo_revisions = get_repo_revision_history(
+self.settings["EROOT"],
+[self.settings.repositories[repo_name] for repo_name in
repo_revisions],
+)
+header_repo_revisions = (
+json.loads(header["REPO_REVISIONS"]) if
header.get("REPO_REVISIONS") else {}
+)
+for repo_name, repo_revision in repo_revisions.items():
+rev_list = synced_repo_revisions.get(repo_name, [])
+header_rev = header_repo_revisions.get(repo_name)
+if not rev_list or header_rev in (repo_revision, rev_list[0]):
+continue
+try:
+header_rev_index = (
+None if header_rev is None else rev_list.index(header_rev)
+)
+except ValueError:
+header_rev_index = None
+try:
+repo_revision_index = rev_list.index(repo_revision)
+except ValueError:
+repo_revision_index = None
+if re
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/repository/, lib/portage/sync/modules/mercurial/, ...
commit: cdf328f5629c20ecf792a6200c8c2c24d932ee68
Author: Mike Gilbert gentoo org>
AuthorDate: Mon May 13 19:21:47 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue May 21 17:27:30 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cdf328f5
Drop portage.util.shlex_split
This has been a trivial wrapper since we dropped support for python2.
Signed-off-by: Mike Gilbert gentoo.org>
bin/dispatch-conf | 5 +++--
bin/ebuild | 5 ++---
bin/egencache | 5 ++---
bin/quickpkg| 9 +
lib/_emerge/BinpkgExtractorAsync.py | 14 +-
lib/_emerge/BinpkgFetcher.py| 4 ++--
lib/_emerge/actions.py | 3 ++-
lib/_emerge/main.py | 3 ++-
lib/portage/_emirrordist/FetchTask.py | 3 ++-
lib/portage/_sets/dbapi.py | 6 +++---
lib/portage/_sets/libs.py | 4 ++--
lib/portage/dbapi/bintree.py| 5 ++---
lib/portage/dbapi/porttree.py | 5 +++--
lib/portage/dbapi/vartree.py| 17 ++---
lib/portage/dispatch_conf.py| 5 +++--
lib/portage/emaint/modules/logs/logs.py | 5 +++--
lib/portage/emaint/modules/sync/sync.py | 3 ++-
lib/portage/getbinpkg.py| 3 ++-
lib/portage/gpg.py | 7 ---
lib/portage/gpkg.py | 11 ++-
lib/portage/package/ebuild/_config/LocationsManager.py | 4 ++--
.../package/ebuild/_config/env_var_validation.py| 4 ++--
.../package/ebuild/_parallel_manifest/ManifestTask.py | 5 +++--
lib/portage/package/ebuild/config.py| 6 +++---
lib/portage/package/ebuild/doebuild.py | 9 -
lib/portage/package/ebuild/fetch.py | 6 +++---
lib/portage/repository/config.py| 4 ++--
lib/portage/sync/modules/git/git.py | 11 ++-
lib/portage/sync/modules/mercurial/mercurial.py | 15 ---
lib/portage/sync/modules/rsync/rsync.py | 9 +++--
lib/portage/tests/ebuild/test_fetch.py | 5 +++--
lib/portage/tests/emerge/conftest.py| 7 ---
lib/portage/tests/emerge/test_config_protect.py | 5 +++--
lib/portage/util/ExtractKernelVersion.py| 5 +++--
lib/portage/util/__init__.py| 11 +--
lib/portage/util/_async/BuildLogger.py | 4 ++--
lib/portage/util/_dyn_libs/soname_deps.py | 6 +++---
37 files changed, 116 insertions(+), 122 deletions(-)
diff --git a/bin/dispatch-conf b/bin/dispatch-conf
index e34e9587f7..93164d909e 100755
--- a/bin/dispatch-conf
+++ b/bin/dispatch-conf
@@ -14,6 +14,7 @@
import atexit
import errno
import re
+import shlex
import subprocess
import sys
import termios
@@ -89,7 +90,7 @@ def cmd_var_is_valid(cmd):
Return true if the first whitespace-separated token contained
in cmd is an executable file, false otherwise.
"""
-cmd = portage.util.shlex_split(cmd)
+cmd = shlex.split(cmd)
if not cmd:
return False
@@ -130,7 +131,7 @@ class dispatch:
if pager is None or not cmd_var_is_valid(pager):
pager = "cat"
-pager_basename = os.path.basename(portage.util.shlex_split(pager)[0])
+pager_basename = os.path.basename(shlex.split(pager)[0])
if pager_basename == "less":
less_opts = self.options.get("less-opts")
if less_opts is not None and less_opts.strip():
diff --git a/bin/ebuild b/bin/ebuild
index 043e5bc476..113a6214d6 100755
--- a/bin/ebuild
+++ b/bin/ebuild
@@ -34,6 +34,7 @@ signal.signal(signal.SIGUSR1, debug_signal)
import argparse
from os import path as osp
+import shlex
import sys
import textwrap
@@ -107,9 +108,7 @@ def main():
parser.error("missing required args")
if not opts.ignore_default_opts:
-default_opts = portage.util.shlex_split(
-portage.settings.get("EBUILD_DEFAULT_OPTS", "")
-)
+default_opts = shlex.split(portage.settings.get("EBUILD_DEFAULT_OPTS",
""))
opts, pargs = parser.parse_known_args(default_opts + sys.argv[1:])
debug = opts.debug
diff --git a/bin/egencache b/bin/egencache
index dbe8d27fee..36477e1abf 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -33,6 +33,7 @@ try:
signal.signal(signal.SIGUSR1, debug_signal)
import argparse
+import shlex
import stat
import sys
import f
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/_emerge/, lib/portage/, bin/, ...
commit: a33065dab4bebd476d0dcba3c1659fbe0e515469
Author: Mike Gilbert gentoo org>
AuthorDate: Wed May 15 17:25:18 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue May 21 17:24:00 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a33065da
Do not use shlex.split for CONFIG_PROTECT
PMS says this is a whitespace-separated list, so we should not treat it
as a shell expression.
Signed-off-by: Mike Gilbert gentoo.org>
bin/dispatch-conf | 4 ++--
bin/portageq| 8
lib/_emerge/depgraph.py | 6 +++---
lib/_emerge/post_emerge.py | 2 +-
lib/portage/_global_updates.py | 6 +++---
lib/portage/dbapi/vartree.py| 8
lib/portage/emaint/modules/sync/sync.py | 4 +---
7 files changed, 18 insertions(+), 20 deletions(-)
diff --git a/bin/dispatch-conf b/bin/dispatch-conf
index 601110ce87..e34e9587f7 100755
--- a/bin/dispatch-conf
+++ b/bin/dispatch-conf
@@ -202,7 +202,7 @@ class dispatch:
protect_obj = portage.util.ConfigProtect(
config_root,
config_paths,
-
portage.util.shlex_split(portage.settings.get("CONFIG_PROTECT_MASK", "")),
+portage.settings.get("CONFIG_PROTECT_MASK", "").split(),
case_insensitive=("case-insensitive-fs" in
portage.settings.features),
)
@@ -616,4 +616,4 @@ if len(sys.argv) > 1:
# for testing
d.grind(sys.argv[1:])
else:
-d.grind(portage.util.shlex_split(portage.settings.get("CONFIG_PROTECT",
"")))
+d.grind(portage.settings.get("CONFIG_PROTECT", "").split())
diff --git a/bin/portageq b/bin/portageq
index 93fa4edeba..9ef0cb7d62 100755
--- a/bin/portageq
+++ b/bin/portageq
@@ -410,8 +410,8 @@ try:
from portage.util import ConfigProtect
settings = portage.settings
-protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))
-protect_mask =
portage.util.shlex_split(settings.get("CONFIG_PROTECT_MASK", ""))
+protect = settings.get("CONFIG_PROTECT", "").split()
+protect_mask = settings.get("CONFIG_PROTECT_MASK", "").split()
protect_obj = ConfigProtect(
root,
protect,
@@ -449,8 +449,8 @@ try:
from portage.util import ConfigProtect
settings = portage.settings
-protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))
-protect_mask =
portage.util.shlex_split(settings.get("CONFIG_PROTECT_MASK", ""))
+protect = settings.get("CONFIG_PROTECT", "").split()
+protect_mask = settings.get("CONFIG_PROTECT_MASK", "").split()
protect_obj = ConfigProtect(
root,
protect,
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 9673d85f87..13add990e6 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -55,7 +55,7 @@ from portage.package.ebuild.getmaskingstatus import
_getmaskingstatus, _MaskReas
from portage._sets import SETPREFIX
from portage._sets.base import InternalPackageSet
from portage.dep._slot_operator import evaluate_slot_operator_equal_deps
-from portage.util import ConfigProtect, shlex_split, new_protect_filename
+from portage.util import ConfigProtect, new_protect_filename
from portage.util import cmp_sort_key, writemsg, writemsg_stdout
from portage.util import ensure_dirs, normalize_path
from portage.util import writemsg_level, write_atomic
@@ -10650,8 +10650,8 @@ class depgraph:
settings = self._frozen_config.roots[root].settings
protect_obj[root] = ConfigProtect(
settings["PORTAGE_CONFIGROOT"],
-shlex_split(settings.get("CONFIG_PROTECT", "")),
-shlex_split(settings.get("CONFIG_PROTECT_MASK", "")),
+settings.get("CONFIG_PROTECT", "").split(),
+settings.get("CONFIG_PROTECT_MASK", "").split(),
case_insensitive=("case-insensitive-fs" in
settings.features),
)
diff --git a/lib/_emerge/post_emerge.py b/lib/_emerge/post_emerge.py
index 37e2c3cc80..6f1f1c243d 100644
--- a/lib/_emerge/post_emerge.py
+++ b/lib/_emerge/post_emerge.py
@@ -93,7 +93,7 @@ def post_emerge(myaction, myopts, myfiles, target_root,
trees, mtimedb, retval):
settings.regenerate()
settings.lock()
-config_protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT",
""))
+config_protect = settings.get("CONFIG_PROTECT", "").split()
infodirs = settings.get("INFOPATH", "").split(":") + settings.get(
"INFODIR", ""
).split(":")
diff --git a/lib/portage/_global_updates.py b/lib/portage/_global_updates.py
index f7997fc37c..4ed8a3f9a7 100644
--- a/lib/portage/_global_updates.py
+++ b/lib/portage/_global_updates.py
@@ -15,7 +15,7 @@ from portage.update import (
update_config_files,
update_dbentry,
)
-from portage.util import
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/git/
commit: 95b4337d376a146db8fda7717393366175cbd285
Author: Alex Xu (Hello71) yahoo ca>
AuthorDate: Sun May 5 20:24:52 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun May 12 16:20:02 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=95b4337d
sync: git: add safe.directory for get head commit
Closes: https://bugs.gentoo.org/930992
Fixes: 1339a02103 ("sync: git: include signing key and git revision in log
output")
Signed-off-by: Alex Xu (Hello71) yahoo.ca>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/git/git.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/portage/sync/modules/git/git.py
b/lib/portage/sync/modules/git/git.py
index 8fdbf97de0..a2830280fb 100644
--- a/lib/portage/sync/modules/git/git.py
+++ b/lib/portage/sync/modules/git/git.py
@@ -606,6 +606,7 @@ class GitSync(NewBase):
if self.bin_command is None:
# return quietly so that we don't pollute emerge --info output
return (1, False)
+self.add_safe_directory()
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
try:
ret = (
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: 8c6e5d06afbf6fca1893cff5ed777e44f93b7a5d
Author: Alexey Gladkov kernel org>
AuthorDate: Sun Mar 3 16:41:08 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8c6e5d06
sync/zipfile: Handle ETag header
Most services add an ETag header and determine whether the locally
cached version of the URL has expired. So we can add ETag processing to
avoid unnecessary downloading and unpacking of the zip archive.
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/zipfile.py | 36 +++--
1 file changed, 29 insertions(+), 7 deletions(-)
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
index 1762d2c8f1..bb78b39243 100644
--- a/lib/portage/sync/modules/zipfile/zipfile.py
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -10,7 +10,7 @@ import tempfile
import urllib.request
import portage
-from portage.util import writemsg_level
+from portage.util import writemsg_level, writemsg_stdout
from portage.sync.syncbase import SyncBase
@@ -31,13 +31,31 @@ class ZipFile(SyncBase):
if kwargs:
self._kwargs(kwargs)
-# initial checkout
-zip_uri = self.repo.sync_uri
+req = urllib.request.Request(url=self.repo.sync_uri)
-with urllib.request.urlopen(zip_uri) as response:
-with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
-shutil.copyfileobj(response, tmp_file)
-zip_file = tmp_file.name
+info = portage.grabdict(os.path.join(self.repo.location, ".info"))
+if "etag" in info:
+req.add_header("If-None-Match", info["etag"][0])
+
+try:
+with urllib.request.urlopen(req) as response:
+with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+shutil.copyfileobj(response, tmp_file)
+
+zip_file = tmp_file.name
+etag = response.headers.get("etag")
+
+except urllib.error.HTTPError as resp:
+if resp.code == 304:
+writemsg_stdout(">>> The repository has not changed.\n",
noiselevel=-1)
+return (os.EX_OK, False)
+
+writemsg_level(
+f"!!! Unable to obtain zip archive: {resp}\n",
+noiselevel=-1,
+level=logging.ERROR,
+)
+return (1, False)
if not zipfile.is_zipfile(zip_file):
msg = "!!! file is not a zip archive."
@@ -77,6 +95,10 @@ class ZipFile(SyncBase):
with open(dstpath, "wb") as dstfile:
shutil.copyfileobj(srcfile, dstfile)
+with open(os.path.join(self.repo.location, ".info"), "w") as infofile:
+if etag:
+infofile.write(f"etag {etag}\n")
+
os.unlink(zip_file)
return (os.EX_OK, True)
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: 7e93192fda22594b9e9d223c54a39b4bad0554f9
Author: Alexey Gladkov kernel org>
AuthorDate: Mon Mar 11 00:25:07 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7e93192f
sync/zipfile: Add retrieve_head to return archive checksum
Since we have an ETag, we can return the checksum of the archive. It
will be a replacement for the head commit of the repository.
Suggested-by: Zac Medico gentoo.org>
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/__init__.py | 3 ++-
lib/portage/sync/modules/zipfile/zipfile.py | 9 +
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/lib/portage/sync/modules/zipfile/__init__.py
b/lib/portage/sync/modules/zipfile/__init__.py
index 19fe3af412..e44833088c 100644
--- a/lib/portage/sync/modules/zipfile/__init__.py
+++ b/lib/portage/sync/modules/zipfile/__init__.py
@@ -21,10 +21,11 @@ module_spec = {
"sourcefile": "zipfile",
"class": "ZipFile",
"description": doc,
-"functions": ["sync"],
+"functions": ["sync", "retrieve_head"],
"func_desc": {
"sync": "Performs an archived http download of the "
+ "repository, then unpacks it.",
+"retrieve_head": "Returns the checksum of the unpacked
archive.",
},
"validate_config": CheckSyncConfig,
"module_specific_options": (),
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
index bb78b39243..3cd210a64b 100644
--- a/lib/portage/sync/modules/zipfile/zipfile.py
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -26,6 +26,15 @@ class ZipFile(SyncBase):
def __init__(self):
SyncBase.__init__(self, "emerge", ">=sys-apps/portage-2.3")
+def retrieve_head(self, **kwargs):
+"""Get information about the checksum of the unpacked archive"""
+if kwargs:
+self._kwargs(kwargs)
+info = portage.grabdict(os.path.join(self.repo.location, ".info"))
+if "etag" in info:
+return (os.EX_OK, info["etag"][0])
+return (1, False)
+
def sync(self, **kwargs):
"""Sync the repository"""
if kwargs:
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: ced2e6d4f4ac95b8e17cf7dae964a64037a85bf0
Author: Alexey Gladkov kernel org>
AuthorDate: Mon Mar 11 17:09:05 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:09 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ced2e6d4
sync/zipfile: Recycle files that have not changed
We can check whether the content of files from the archive differs from
the current revision. This will give us several advantages:
* This will give us some meaning to the mtime of files, since it will
prevent the timestamps of unmodified files from being changed.
* This will also get rid of recreatiing self.repo.location, which will
allow sync with FEATURES=usersync because self.repo.location is reused.
Suggested-by: Zac Medico gentoo.org>
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/zipfile.py | 32 -
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
index 3cd210a64b..edfb5aa681 100644
--- a/lib/portage/sync/modules/zipfile/zipfile.py
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -35,6 +35,16 @@ class ZipFile(SyncBase):
return (os.EX_OK, info["etag"][0])
return (1, False)
+def _do_cmp(self, f1, f2):
+bufsize = 8 * 1024
+while True:
+b1 = f1.read(bufsize)
+b2 = f2.read(bufsize)
+if b1 != b2:
+return False
+if not b1:
+return True
+
def sync(self, **kwargs):
"""Sync the repository"""
if kwargs:
@@ -76,7 +86,15 @@ class ZipFile(SyncBase):
return (1, False)
# Drop previous tree
-shutil.rmtree(self.repo.location)
+tempdir = tempfile.mkdtemp(prefix=".temp", dir=self.repo.location)
+tmpname = os.path.basename(tempdir)
+
+for name in os.listdir(self.repo.location):
+if name != tmpname:
+os.rename(
+os.path.join(self.repo.location, name),
+os.path.join(tempdir, name),
+)
with zipfile.ZipFile(zip_file) as archive:
strip_comp = 0
@@ -101,9 +119,21 @@ class ZipFile(SyncBase):
continue
with archive.open(n) as srcfile:
+prvpath = os.path.join(tempdir, *parts[strip_comp:])
+
+if os.path.exists(prvpath):
+with open(prvpath, "rb") as prvfile:
+if self._do_cmp(prvfile, srcfile):
+os.rename(prvpath, dstpath)
+continue
+srcfile.seek(0)
+
with open(dstpath, "wb") as dstfile:
shutil.copyfileobj(srcfile, dstfile)
+# Drop previous tree
+shutil.rmtree(tempdir)
+
with open(os.path.join(self.repo.location, ".info"), "w") as infofile:
if etag:
infofile.write(f"etag {etag}\n")
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/sync/
commit: b01cd4208a17a141311d490788aff11537312575
Author: Zac Medico gentoo org>
AuthorDate: Tue Mar 12 10:20:56 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:09 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b01cd420
sync/zipfile: Add testcase for etag
Signed-off-by: Zac Medico gentoo.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/tests/sync/meson.build | 1 +
lib/portage/tests/sync/test_sync_zipfile.py | 99 +
2 files changed, 100 insertions(+)
diff --git a/lib/portage/tests/sync/meson.build
b/lib/portage/tests/sync/meson.build
index b78583021f..8c566080e3 100644
--- a/lib/portage/tests/sync/meson.build
+++ b/lib/portage/tests/sync/meson.build
@@ -1,6 +1,7 @@
py.install_sources(
[
'test_sync_local.py',
+'test_sync_zipfile.py',
'__init__.py',
'__test__.py',
],
diff --git a/lib/portage/tests/sync/test_sync_zipfile.py
b/lib/portage/tests/sync/test_sync_zipfile.py
new file mode 100644
index 00..4fbde8a351
--- /dev/null
+++ b/lib/portage/tests/sync/test_sync_zipfile.py
@@ -0,0 +1,99 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import http.server
+import os
+import shutil
+import socketserver
+import subprocess
+import tempfile
+import textwrap
+import threading
+from functools import partial
+
+import portage
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import ResolverPlayground
+
+
+class test_sync_zipfile_case(TestCase):
+def test_sync_zipfile(self):
+cpv = "dev-libs/A-0"
+ebuilds = {
+cpv: {"EAPI": "8"},
+}
+etag = "foo"
+
+server = None
+playground = None
+tmpdir = tempfile.mkdtemp()
+try:
+
+class Handler(http.server.SimpleHTTPRequestHandler):
+def end_headers(self):
+self.send_header("etag", etag)
+super().end_headers()
+
+server = socketserver.TCPServer(
+("127.0.0.1", 0),
+partial(Handler, directory=tmpdir),
+)
+threading.Thread(target=server.serve_forever, daemon=True).start()
+
+playground = ResolverPlayground(
+ebuilds=ebuilds,
+)
+settings = playground.settings
+
+env = settings.environ()
+
+repos_conf = textwrap.dedent(
+"""
+[test_repo]
+location = %(location)s
+sync-type = zipfile
+sync-uri = %(sync-uri)s
+auto-sync = true
+"""
+)
+
+repo_location = f"{playground.eprefix}/var/repositories/test_repo"
+
+env["PORTAGE_REPOSITORIES"] = repos_conf % {
+"location": repo_location,
+"sync-uri":
"http://{}:{}/test_repo.zip".format(*server.server_address),
+}
+
+shutil.make_archive(os.path.join(tmpdir, "test_repo"), "zip",
repo_location)
+
+ebuild =
playground.trees[playground.eroot]["porttree"].dbapi.findname(cpv)
+self.assertTrue(os.path.exists(ebuild))
+shutil.rmtree(repo_location)
+self.assertFalse(os.path.exists(ebuild))
+
+result = subprocess.run(
+[
+"emerge",
+"--sync",
+],
+env=env,
+stdout=subprocess.PIPE,
+stderr=subprocess.STDOUT,
+)
+output = result.stdout.decode(errors="replace")
+try:
+self.assertEqual(result.returncode, os.EX_OK)
+except Exception:
+print(output)
+raise
+
+repo = settings.repositories["test_repo"]
+sync_mod = portage.sync.module_controller.get_class("zipfile")
+status, repo_revision = sync_mod().retrieve_head(options={"repo":
repo})
+self.assertEqual(status, os.EX_OK)
+self.assertEqual(repo_revision, etag)
+finally:
+if server is not None:
+server.shutdown()
+shutil.rmtree(tmpdir)
+playground.cleanup()
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: 80445d9b00bfcd1eb4955cf3ecb397b4c02663ba
Author: Alexey Gladkov kernel org>
AuthorDate: Mon Feb 12 13:59:40 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=80445d9b
sync: Add method to download zip archives
Add a simple method for synchronizing repository as a snapshot in a zip
archive. The implementation does not require external utilities to
download and unpack archive. This makes the method very cheap.
The main usecase being considered is obtaining snapshots of github
repositories, but many other web interfaces for git also support
receiving snapshots in zip format.
For example, to get a snapshot of the master branch:
https://github.com/gentoo/portage/archive/refs/heads/master.zip
https://gitweb.gentoo.org/proj/portage.git/snapshot/portage-master.zip
or a link to a snapshot of the tag:
https://github.com/gentoo/portage/archive/refs/tags/portage-3.0.61.zip
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/__init__.py | 33 +++
lib/portage/sync/modules/zipfile/zipfile.py | 82
2 files changed, 115 insertions(+)
diff --git a/lib/portage/sync/modules/zipfile/__init__.py
b/lib/portage/sync/modules/zipfile/__init__.py
new file mode 100644
index 00..19fe3af412
--- /dev/null
+++ b/lib/portage/sync/modules/zipfile/__init__.py
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2024 Alexey Gladkov
+
+doc = """Zipfile plug-in module for portage.
+Performs a http download of a portage snapshot and unpacks it to the repo
+location."""
+__doc__ = doc[:]
+
+
+import os
+
+from portage.sync.config_checks import CheckSyncConfig
+
+
+module_spec = {
+"name": "zipfile",
+"description": doc,
+"provides": {
+"zipfile-module": {
+"name": "zipfile",
+"sourcefile": "zipfile",
+"class": "ZipFile",
+"description": doc,
+"functions": ["sync"],
+"func_desc": {
+"sync": "Performs an archived http download of the "
++ "repository, then unpacks it.",
+},
+"validate_config": CheckSyncConfig,
+"module_specific_options": (),
+},
+},
+}
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
new file mode 100644
index 00..1762d2c8f1
--- /dev/null
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2024 Alexey Gladkov
+
+import os
+import os.path
+import logging
+import zipfile
+import shutil
+import tempfile
+import urllib.request
+
+import portage
+from portage.util import writemsg_level
+from portage.sync.syncbase import SyncBase
+
+
+class ZipFile(SyncBase):
+"""ZipFile sync module"""
+
+short_desc = "Perform sync operations on GitHub repositories"
+
+@staticmethod
+def name():
+return "ZipFile"
+
+def __init__(self):
+SyncBase.__init__(self, "emerge", ">=sys-apps/portage-2.3")
+
+def sync(self, **kwargs):
+"""Sync the repository"""
+if kwargs:
+self._kwargs(kwargs)
+
+# initial checkout
+zip_uri = self.repo.sync_uri
+
+with urllib.request.urlopen(zip_uri) as response:
+with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+shutil.copyfileobj(response, tmp_file)
+zip_file = tmp_file.name
+
+if not zipfile.is_zipfile(zip_file):
+msg = "!!! file is not a zip archive."
+self.logger(self.xterm_titles, msg)
+writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
+
+os.unlink(zip_file)
+
+return (1, False)
+
+# Drop previous tree
+shutil.rmtree(self.repo.location)
+
+with zipfile.ZipFile(zip_file) as archive:
+strip_comp = 0
+
+for f in archive.namelist():
+f = os.path.normpath(f)
+if os.path.basename(f) == "profiles":
+strip_comp = f.count("/")
+break
+
+for n in archive.infolist():
+p = os.path.normpath(n.filename)
+
+if os.path.isabs(p):
+continue
+
+parts = p.split("/")
+dstpath = os.path.join(self.repo.location, *parts[strip_comp:])
+
+if n.is_dir():
+os.makedirs(dstpath, mode=0o755, exist_ok=True)
+continue
+
+with archive.open(n) as srcfile:
+with open(dstpath, "wb") as dstfile:
+shutil.copyfileobj(srcfile, dstfile)
+
+os.unlink(zip_file)
+
+return (os.EX_OK, True)
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 36235596e061bf8cf4729b3915f9aaa6ec80baa3
Author: Alfred Wingate protonmail com>
AuthorDate: Sat Apr 6 08:28:43 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Apr 26 22:05:48 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=36235596
lib: adhere to python package version conventions
* Commit metadata isn't valid version that python tooling is expected to
parse. Follow python ecosystem conventions and make it a local
version.
https://packaging.python.org/en/latest/specifications/version-specifiers/#local-version-segments
Example:
* Old: 3.0.63-g08a2bc380
* New: 3.0.63+g08a2bc380
Bug: https://bugs.gentoo.org/926966
Signed-off-by: Alfred Wingate protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1314
Signed-off-by: Sam James gentoo.org>
lib/portage/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index a468eeaff3..21bf993170 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -732,7 +732,7 @@ if installation.TYPE == installation.TYPES.SOURCE:
output = _unicode_decode(proc.communicate()[0],
encoding=encoding)
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
-VERSION = output.lstrip('portage-').strip()
+VERSION = output.lstrip("portage-").strip().replace("-g",
"+g")
else:
VERSION = "HEAD"
return VERSION
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 381fad5e3554ec94ec5626e8c17874f32b30b752
Author: Sam James gentoo org>
AuthorDate: Tue Aug 29 07:26:36 2023 +
Commit: Sam James gentoo org>
CommitDate: Fri Apr 26 22:05:48 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=381fad5e
lib: use more pure git-describe output for --version
Use `git describe --dirty` output rather than mangling git-describe and
reinventing
--dirty by manually checking for changes post-commit.
We no longer mangle the 7th commit post-tag into _p7, but instead do:
${tag}-7-${last_commit}.
This is similar to gnulib's git-version-gen (which we may still want to import,
not sure, this seems enough for now) and is familiar output for developers.
Example:
* Old: 3.0.51_p7
* New: 3.0.51-7-g098b30548
Bug: https://bugs.gentoo.org/912209
Signed-off-by: Sam James gentoo.org>
lib/portage/__init__.py | 35 ---
1 file changed, 4 insertions(+), 31 deletions(-)
diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index aa81bdb4c2..a468eeaff3 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -720,10 +720,7 @@ if installation.TYPE == installation.TYPES.SOURCE:
BASH_BINARY,
"-c",
(
-f"cd {_shell_quote(PORTAGE_BASE_PATH)} ; git describe
--match 'portage-*' || exit $? ; "
-'if [ -n "`git diff-index --name-only --diff-filter=M
HEAD`" ] ; '
-"then echo modified ; git rev-list --format=%%ct -n 1
HEAD ; fi ; "
-"exit 0"
+f"cd {_shell_quote(PORTAGE_BASE_PATH)} ; git describe
--dirty --match 'portage-*' || exit $? ; "
),
]
cmd = [
@@ -735,33 +732,9 @@ if installation.TYPE == installation.TYPES.SOURCE:
output = _unicode_decode(proc.communicate()[0],
encoding=encoding)
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
-output_lines = output.splitlines()
-if output_lines:
-version_split = output_lines[0].split("-")
-if len(version_split) > 1:
-VERSION = version_split[1]
-patchlevel = False
-if len(version_split) > 2:
-patchlevel = True
-VERSION = f"{VERSION}_p{version_split[2]}"
-if len(output_lines) > 1 and output_lines[1] ==
"modified":
-head_timestamp = None
-if len(output_lines) > 3:
-try:
-head_timestamp = int(output_lines[3])
-except ValueError:
-pass
-timestamp = int(time.time())
-if (
-head_timestamp is not None
-and timestamp > head_timestamp
-):
-timestamp = timestamp - head_timestamp
-if not patchlevel:
-VERSION = f"{VERSION}_p0"
-VERSION = f"{VERSION}_p{timestamp}"
-return VERSION
-VERSION = "HEAD"
+VERSION = output.lstrip('portage-').strip()
+else:
+VERSION = "HEAD"
return VERSION
VERSION = _LazyVersion()
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/
commit: 72f41d07b5396195a98691fafeb3e5b207a76564 Author: Raul E Rangel chromium org> AuthorDate: Thu Mar 21 20:29:59 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 24 22:18:34 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=72f41d07 Reapply "config: Don't directly modify FEATURES" This reverts commit b150419d28bd7afb98404a829c639584d34efc03. It turns out we need to keep the open coded version to avoid creating a persistent setting. See https://github.com/gentoo/portage/pull/1098#issuecomment-1761638611 This change just adds a sorted() so we get deterministic ordering. Bug: https://bugs.gentoo.org/914441 Signed-off-by: Raul E Rangel chromium.org> Closes: https://github.com/gentoo/portage/pull/1312 Signed-off-by: Zac Medico gentoo.org> lib/portage/package/ebuild/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py index bafdc55a08..67fd1bb18d 100644 --- a/lib/portage/package/ebuild/config.py +++ b/lib/portage/package/ebuild/config.py @@ -2206,7 +2206,9 @@ class config: # "test" is in IUSE and USE=test is masked, so execution # of src_test() probably is not reliable. Therefore, # temporarily disable FEATURES=test just for this package. -self["FEATURES"] = " ".join(x for x in self.features if x != "test") +self["FEATURES"] = " ".join( +x for x in sorted(self.features) if x != "test" +) # Allow _* flags from USE_EXPAND wildcards to pass through here. use.difference_update(
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/util/file_copy/, src/
commit: 23529ee81964665107400e87fc3d49c256e915c0
Author: Mike Gilbert gentoo org>
AuthorDate: Fri Mar 1 15:45:58 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 15 20:05:34 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=23529ee8
Replace linux_reflink extension module
Python 3.8 added support for copy_file_range in the os module,
so we can just call that directly.
Also, we can use the FICLONE ioctl for fast file clones on supported
filesystems (btrfs).
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/file_copy.py | 137 ++
lib/portage/util/file_copy/__init__.py | 36 ---
lib/portage/util/file_copy/meson.build | 7 -
lib/portage/util/meson.build | 2 +-
src/meson.build| 20 --
src/portage_util_file_copy_reflink_linux.c | 396 -
6 files changed, 138 insertions(+), 460 deletions(-)
diff --git a/lib/portage/util/file_copy.py b/lib/portage/util/file_copy.py
new file mode 100644
index 00..e3926d8ef6
--- /dev/null
+++ b/lib/portage/util/file_copy.py
@@ -0,0 +1,137 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import fcntl
+import logging
+import os
+import platform
+import shutil
+import sys
+
+
+logger = logging.getLogger(__name__)
+
+# Added in Python 3.12
+FICLONE = getattr(fcntl, "FICLONE", 0x40049409)
+
+# Unavailable in PyPy
+SEEK_DATA = getattr(os, "SEEK_DATA", 3)
+SEEK_HOLE = getattr(os, "SEEK_HOLE", 4)
+
+
+def _get_chunks(src):
+try:
+offset_hole = 0
+while True:
+try:
+# Find the next bit of data
+offset_data = os.lseek(src, offset_hole, SEEK_DATA)
+except OSError as e:
+# Re-raise for unexpected errno values
+if e.errno not in (errno.EINVAL, errno.ENXIO):
+raise
+
+offset_end = os.lseek(src, 0, os.SEEK_END)
+
+if e.errno == errno.ENXIO:
+# End of file
+if offset_end > offset_hole:
+# Hole at end of file
+yield (offset_end, 0)
+else:
+# SEEK_DATA failed with EINVAL, return the whole file
+yield (0, offset_end)
+
+break
+else:
+offset_hole = os.lseek(src, offset_data, SEEK_HOLE)
+yield (offset_data, offset_hole - offset_data)
+
+except OSError:
+logger.warning("_get_chunks failed unexpectedly",
exc_info=sys.exc_info())
+raise
+
+
+def _do_copy_file_range(src, dst, offset, count):
+while count > 0:
+# count must fit in ssize_t
+c = min(count, sys.maxsize)
+written = os.copy_file_range(src, dst, c, offset, offset)
+if written == 0:
+# https://bugs.gentoo.org/828844
+raise OSError(errno.EOPNOTSUPP, os.strerror(errno.EOPNOTSUPP))
+offset += written
+count -= written
+
+
+def _do_sendfile(src, dst, offset, count):
+os.lseek(dst, offset, os.SEEK_SET)
+while count > 0:
+# count must fit in ssize_t
+c = min(count, sys.maxsize)
+written = os.sendfile(dst, src, offset, c)
+offset += written
+count -= written
+
+
+def _fastcopy(src, dst):
+with (
+open(src, "rb", buffering=0) as srcf,
+open(dst, "wb", buffering=0) as dstf,
+):
+srcfd = srcf.fileno()
+dstfd = dstf.fileno()
+
+if platform.system() == "Linux":
+try:
+fcntl.ioctl(dstfd, FICLONE, srcfd)
+return
+except OSError:
+pass
+
+try_cfr = hasattr(os, "copy_file_range")
+
+for offset, count in _get_chunks(srcfd):
+if count == 0:
+os.ftruncate(dstfd, offset)
+else:
+if try_cfr:
+try:
+_do_copy_file_range(srcfd, dstfd, offset, count)
+continue
+except OSError as e:
+try_cfr = False
+if e.errno not in (errno.EXDEV, errno.ENOSYS,
errno.EOPNOTSUPP):
+logger.warning(
+"_do_copy_file_range failed unexpectedly",
+exc_info=sys.exc_info(),
+)
+try:
+_do_sendfile(srcfd, dstfd, offset, count)
+except OSError:
+logger.warning(
+"_do_sendfile failed unexpectedly",
exc_info=sys.exc_info()
+)
+raise
+
+
+def copyfile(src, dst):
+"""
+Copy the contents (no metadata) of the file named src to a file
+named dst.
+
+If possible, cop
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/file_copy/
commit: 0cc3f9e269b26173c2f81d731c5fe208b758270b
Author: Mike Gilbert gentoo org>
AuthorDate: Sun Mar 3 02:28:42 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 15 20:05:34 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0cc3f9e2
Improve testCopyFileSparse
Actually create sparse blocks at the start and end.
Check file size before/after copying.
Ensure sparse output when _fastcopy succeeds.
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/tests/util/file_copy/test_copyfile.py | 35 ---
1 file changed, 24 insertions(+), 11 deletions(-)
diff --git a/lib/portage/tests/util/file_copy/test_copyfile.py
b/lib/portage/tests/util/file_copy/test_copyfile.py
index e91a47bed8..e114e6ae35 100644
--- a/lib/portage/tests/util/file_copy/test_copyfile.py
+++ b/lib/portage/tests/util/file_copy/test_copyfile.py
@@ -3,13 +3,14 @@
import shutil
import tempfile
+from unittest.mock import patch
import pytest
from portage import os
from portage.tests import TestCase
from portage.checksum import perform_md5
-from portage.util.file_copy import copyfile
+from portage.util.file_copy import copyfile, _fastcopy
class CopyFileTestCase(TestCase):
@@ -42,25 +43,37 @@ class CopyFileSparseTestCase(TestCase):
# files too big, in case the filesystem doesn't support
# sparse files.
with open(src_path, "wb") as f:
+f.seek(2**16, os.SEEK_SET)
f.write(content)
-f.seek(2**17, 1)
-f.write(content)
-f.seek(2**18, 1)
+f.seek(2**17, os.SEEK_SET)
f.write(content)
# Test that sparse blocks are handled correctly at
-# the end of the file (involves seek and truncate).
-f.seek(2**17, 1)
+# the end of the file.
+f.truncate(2**18)
-copyfile(src_path, dest_path)
+fastcopy_success = False
+
+def mock_fastcopy(src, dst):
+nonlocal fastcopy_success
+_fastcopy(src, dst)
+fastcopy_success = True
+
+with patch("portage.util.file_copy._fastcopy", new=mock_fastcopy):
+copyfile(src_path, dest_path)
self.assertEqual(perform_md5(src_path), perform_md5(dest_path))
-# This last part of the test is expected to fail when sparse
-# copy is not implemented, so mark it xfail:
-pytest.xfail(reason="sparse copy is not implemented")
+src_stat = os.stat(src_path)
+dest_stat = os.stat(dest_path)
+
+self.assertEqual(src_stat.st_size, dest_stat.st_size)
# If sparse blocks were preserved, then both files should
# consume the same number of blocks.
-self.assertEqual(os.stat(src_path).st_blocks,
os.stat(dest_path).st_blocks)
+# This is expected to fail when sparse copy is not implemented.
+if src_stat.st_blocks != dest_stat.st_blocks:
+if fastcopy_success:
+pytest.fail(reason="sparse copy failed with _fastcopy")
+pytest.xfail(reason="sparse copy is not implemented")
finally:
shutil.rmtree(tempdir)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: b7e89f866a9a1d73ab72670d74e2292b05893849
Author: Mike Gilbert gentoo org>
AuthorDate: Wed Mar 6 04:01:27 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Wed Mar 6 18:19:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7e89f86
util: set a timeout for urlopen calls
A hung urlopen call can cause emerge to produce no output when fetching
binhost data.
Bug: https://bugs.gentoo.org/926221
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/_urlopen.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/portage/util/_urlopen.py b/lib/portage/util/_urlopen.py
index 22f0e08df0..d451a94a89 100644
--- a/lib/portage/util/_urlopen.py
+++ b/lib/portage/util/_urlopen.py
@@ -26,10 +26,10 @@ def have_pep_476():
return hasattr(__import__("ssl"), "_create_unverified_context")
-def urlopen(url, if_modified_since=None, headers={}, proxies=None):
+def urlopen(url, timeout=10, if_modified_since=None, headers={}, proxies=None):
parse_result = urllib_parse.urlparse(url)
if parse_result.scheme not in ("http", "https"):
-return _urlopen(url)
+return _urlopen(url, timeout=timeout)
netloc = parse_result.netloc.rpartition("@")[-1]
url = urllib_parse.urlunparse(
@@ -59,7 +59,7 @@ def urlopen(url, if_modified_since=None, headers={},
proxies=None):
handlers.append(urllib_request.ProxyHandler(proxies))
opener = urllib_request.build_opener(*handlers)
-hdl = opener.open(request)
+hdl = opener.open(request, timeout=timeout)
if hdl.headers.get("last-modified", ""):
try:
add_header = hdl.headers.add_header
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/, lib/portage/util/, lib/portage/tests/util/
commit: 11c65496bd951c3b7778a3c1ea240e347b1f4c38 Author: Zac Medico gentoo org> AuthorDate: Sun Mar 3 21:06:13 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 3 21:10:49 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=11c65496 socks5: Use run_coroutine_exitfuncs() Since commit c3ebdbb42e72 the atexit_register(proxy.stop) call in the get_socks5_proxy function can accept a coroutine function to execute in run_coroutine_exitfuncs(), so convert the ProxyManager stop method to a coroutine and use run_coroutine_exitfuncs(). Bug: https://bugs.gentoo.org/925240 Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/test_socks5.py | 5 ++- lib/portage/util/_eventloop/asyncio_event_loop.py | 12 --- lib/portage/util/socks5.py| 39 +++ 3 files changed, 7 insertions(+), 49 deletions(-) diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py index 4a6d08169d..a8cd0c46c4 100644 --- a/lib/portage/tests/util/test_socks5.py +++ b/lib/portage/tests/util/test_socks5.py @@ -216,10 +216,9 @@ class Socks5ServerTestCase(TestCase): self.assertEqual(result, content) finally: try: -# Also run_exitfuncs to test atexit hook cleanup. -await socks5.proxy.stop() +# Also run_coroutine_exitfuncs to test atexit hook cleanup. self.assertNotEqual(portage.process._exithandlers, []) -portage.process.run_exitfuncs() +await portage.process.run_coroutine_exitfuncs() self.assertEqual(portage.process._exithandlers, []) finally: portage.process._exithandlers = previous_exithandlers diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index a598b1b516..821cc7f102 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -15,7 +15,6 @@ except ImportError: PidfdChildWatcher = None import portage -from portage.util import socks5 class AsyncioEventLoop(_AbstractEventLoop): @@ -75,17 +74,6 @@ class AsyncioEventLoop(_AbstractEventLoop): self._closing = False async def _close_main(self): -# Even though this has an exit hook, invoke it here so that -# we can properly wait for it and avoid messages like this: -# [ERROR] Task was destroyed but it is pending! -if socks5.proxy.is_running(): -# TODO: Convert socks5.proxy.stop() to a regular coroutine -# function so that it doesn't need to be wrapped like this. -async def stop_socks5_proxy(): -await socks5.proxy.stop() - -portage.process.atexit_register(stop_socks5_proxy) - await portage.process.run_coroutine_exitfuncs() portage.process.run_exitfuncs() diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index f8fcdf9fca..c32ba77674 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -6,15 +6,8 @@ import asyncio import errno import os import socket -from typing import Union import portage - -portage.proxy.lazyimport.lazyimport( -globals(), -"portage.util._eventloop.global_event_loop:global_event_loop", -) - import portage.data from portage import _python_interpreter from portage.data import portage_gid, portage_uid, userpriv_groups @@ -65,41 +58,19 @@ class ProxyManager: **spawn_kwargs, ) -def stop(self) -> Union[None, asyncio.Future]: +async def stop(self): """ -Stop the SOCKSv5 server. - -If there is a running asyncio event loop then asyncio.Future is -returned which should be used to wait for the server process -to exit. +Stop the SOCKSv5 server. This method is a coroutine. """ -future = None -try: -loop = asyncio.get_running_loop() -except RuntimeError: -loop = None if self._proc is not None: self._proc.terminate() -if loop is None: -# In this case spawn internals would have used -# portage's global loop when attaching a waiter to -# self._proc, so we are obligated to use that. -global_event_loop().run_until_complete(self._proc.wait()) -else: -if self._proc_waiter is None: -self._proc_waiter = asyncio.ensure_future( -self._proc.wait(), loop=loop -) -future = asyncio.shield(self._proc_waiter) - -if loop is not None and future is None: -future = loop.create_future() -future.set_result(None) +if self._proc_waiter is None: +self._proc_waiter
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/, ...
commit: 6ce2be8d454f95c508d9f547d13487f9de863bdd
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 3 19:53:11 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 3 19:53:11 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6ce2be8d
_validate_deps: Discard configdict["pkg"]["USE"]
Since configdict["pkg"]["USE"] may contain package.use settings
from config.setcpv, it is inappropriate to use here (bug 675748),
so discard it. This is only an issue because configdict["pkg"] is
a sub-optimal place to extract metadata from. This issue does not
necessarily indicate a flaw in the Package constructor, since
passing in precalculated USE can be valid for things like
autounmask USE changes.
Bug: https://bugs.gentoo.org/675748
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/doebuild.py | 12
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 19 ++-
lib/portage/tests/resolver/ResolverPlayground.py | 1 +
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 942fa90101..6691db4e97 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -1813,6 +1813,14 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
invalid_dep_exempt_phases = {"clean", "cleanrm", "help", "prerm", "postrm"}
all_keys = set(Package.metadata_keys)
all_keys.add("SRC_URI")
+# Since configdict["pkg"]["USE"] may contain package.use settings
+# from config.setcpv, it is inappropriate to use here (bug 675748),
+# so discard it. This is only an issue because configdict["pkg"] is
+# a sub-optimal place to extract metadata from. This issue does not
+# necessarily indicate a flaw in the Package constructor, since
+# passing in precalculated USE can be valid for things like
+# autounmask USE changes.
+all_keys.discard("USE")
all_keys = tuple(all_keys)
metadata = mysettings.configdict["pkg"]
if all(k in metadata for k in ("PORTAGE_REPO_NAME", "SRC_URI")):
@@ -1838,6 +1846,10 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
root_config = RootConfig(mysettings, {"porttree": FakeTree(mydbapi)}, None)
+# A USE calculation from setcpv should always be available here because
+# mysettings.mycpv is not None, so use it to prevent redundant setcpv
calls.
+metadata["USE"] = mysettings["PORTAGE_USE"]
+
pkg = Package(
built=False,
cpv=mysettings.mycpv,
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index b38605bb90..445fcf6c4e 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -51,10 +51,23 @@ class DoebuildFdPipesTestCase(TestCase):
ebuilds = {
"app-misct/foo-1": {
"EAPI": "8",
+"IUSE": "+foo +bar",
+"REQUIRED_USE": "|| ( foo bar )",
"MISC_CONTENT": ebuild_body,
}
}
+# Populate configdict["pkg"]["USE"] with something arbitrary in order
+# to try and trigger bug 675748 in doebuild _validate_deps.
+arbitrary_package_use = "baz"
+
+user_config = {
+# In order to trigger bug 675748, package.env must be non-empty,
+# but the referenced env file can be empty.
+"package.env": (f"app-misct/foo {os.devnull}",),
+"package.use": (f"app-misct/foo {arbitrary_package_use}",),
+}
+
# Override things that may be unavailable, or may have portability
# issues when running tests in exotic environments.
# prepstrip - bug #447810 (bash read builtin EINTR problem)
@@ -63,7 +76,7 @@ class DoebuildFdPipesTestCase(TestCase):
self.assertEqual(true_binary is None, False, "true command not found")
dev_null = open(os.devnull, "wb")
-playground = ResolverPlayground(ebuilds=ebuilds)
+playground = ResolverPlayground(ebuilds=ebuilds,
user_config=user_config)
try:
QueryCommand._db = playground.trees
root_config = playground.trees[playground.eroot]["root_config"]
@@ -106,6 +119,10 @@ class DoebuildFdPipesTestCase(TestCase):
)
settings.setcpv(pkg)
+# Demonstrate that settings.configdict["pkg"]["USE"] contains our
arbitrary
+# package.use setting in order to trigger bug 675748.
+self.assertEqual(settings.configdict["pkg"]["USE"],
arbitrary_package_use)
+
# Try to trigger the config.environ() split_LC_ALL assertion for
bug 925863.
settings["LC_ALL"] = "C"
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index c0455415a1..f52a98f8db 100644
[gentoo-commits] proj/portage:master commit in: lib/portage/elog/, lib/portage/, lib/portage/util/_eventloop/
commit: c3ebdbb42e72335ca65335c855a82b99537c7606
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 3 06:30:50 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 3 06:30:50 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c3ebdbb4
elog/mod_custom: Spawn processes in background
Since elog_process is typically called while the event loop is
running, hold references to spawned processes and wait for them
asynchronously, ultimately waiting for them if necessary when
the AsyncioEventLoop _close_main method calls _async_finalize
via portage.process.run_coroutine_exitfuncs().
ConfigProtectTestCase is useful for exercising this code, and
this little make.globals patch can be used to test failure
during finalize with this error message:
!!! PORTAGE_ELOG_COMMAND failed with exitcode 1
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -144 +144,2 @@ PORTAGE_ELOG_CLASSES="log warn error"
-PORTAGE_ELOG_SYSTEM="save_summary:log,warn,error,qa echo"
+PORTAGE_ELOG_SYSTEM="save_summary:log,warn,error,qa echo custom"
+PORTAGE_ELOG_COMMAND="/bin/false"
Bug: https://bugs.gentoo.org/925907
Signed-off-by: Zac Medico gentoo.org>
lib/portage/elog/mod_custom.py| 73 +--
lib/portage/process.py| 29 +
lib/portage/util/_eventloop/asyncio_event_loop.py | 8 ++-
3 files changed, 105 insertions(+), 5 deletions(-)
diff --git a/lib/portage/elog/mod_custom.py b/lib/portage/elog/mod_custom.py
index e0ae77e100..a3e199bcb7 100644
--- a/lib/portage/elog/mod_custom.py
+++ b/lib/portage/elog/mod_custom.py
@@ -1,10 +1,33 @@
# elog/mod_custom.py - elog dispatch module
-# Copyright 2006-2020 Gentoo Authors
+# Copyright 2006-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import types
+
+import portage
import portage.elog.mod_save
import portage.exception
import portage.process
+from portage.util.futures import asyncio
+
+# Since elog_process is typically called while the event loop is
+# running, hold references to spawned processes and wait for them
+# asynchronously, ultimately waiting for them if necessary when
+# the AsyncioEventLoop _close_main method calls _async_finalize
+# via portage.process.run_coroutine_exitfuncs().
+_proc_refs = None
+
+
+def _get_procs() -> list[tuple[portage.process.MultiprocessingProcess,
asyncio.Future]]:
+"""
+Return list of (proc, asyncio.ensure_future(proc.wait())) which is not
+inherited from the parent after fork.
+"""
+global _proc_refs
+if _proc_refs is None or _proc_refs.pid != portage.getpid():
+_proc_refs = types.SimpleNamespace(pid=portage.getpid(), procs=[])
+portage.process.atexit_register(_async_finalize)
+return _proc_refs.procs
def process(mysettings, key, logentries, fulltext):
@@ -18,8 +41,50 @@ def process(mysettings, key, logentries, fulltext):
mylogcmd = mysettings["PORTAGE_ELOG_COMMAND"]
mylogcmd = mylogcmd.replace("${LOGFILE}", elogfilename)
mylogcmd = mylogcmd.replace("${PACKAGE}", key)
-retval = portage.process.spawn_bash(mylogcmd)
-if retval != 0:
+loop = asyncio.get_event_loop()
+proc = portage.process.spawn_bash(mylogcmd, returnproc=True)
+procs = _get_procs()
+procs.append((proc, asyncio.ensure_future(proc.wait(), loop=loop)))
+for index, (proc, waiter) in reversed(list(enumerate(procs))):
+if not waiter.done():
+continue
+del procs[index]
+if waiter.result() != 0:
+raise portage.exception.PortageException(
+f"!!! PORTAGE_ELOG_COMMAND failed with exitcode
{waiter.result()}"
+)
+
+
+async def _async_finalize():
+"""
+Async finalize is preferred, since we can wait for process exit status.
+"""
+procs = _get_procs()
+while procs:
+proc, waiter = procs.pop()
+if (await waiter) != 0:
+raise portage.exception.PortageException(
+f"!!! PORTAGE_ELOG_COMMAND failed with exitcode
{waiter.result()}"
+)
+
+
+def finalize():
+"""
+NOTE: This raises PortageException if there are any processes
+still running, so it's better to use _async_finalize instead
+(invoked via portage.process.run_coroutine_exitfuncs() in
+the AsyncioEventLoop _close_main method).
+"""
+procs = _get_procs()
+while procs:
+proc, waiter = procs.pop()
+if not waiter.done():
+waiter.cancel()
+proc.terminate()
+raise portage.exception.PortageException(
+f"!!! PORTAGE_ELOG_COMMAND was killed after it was found
running in the background (pid {proc.pid})"
+)
+elif waiter.result() != 0:
raise portage.exception.PortageException(
-"!!! PORTAGE_ELOG_COMMAND failed with exitcode %d" % retval
+
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 62ee9bf8c680b2a18713da5bd453e3a771257409
Author: Zac Medico gentoo org>
AuthorDate: Sat Mar 2 22:48:54 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Mar 2 22:50:54 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=62ee9bf8
binarytree._populate_remote: Fix UnboundLocalError for binpkg-request-signature
If an InvalidBinaryPackageFormat exception was raised from
get_binpkg_format for binpkg-request-signature then it
triggered an UnboundLocalError here.
Fixes: 445f10f4214c ("Use binpkg extensions and header to get format")
Bug: https://bugs.gentoo.org/926048
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dbapi/bintree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index f4251b47d6..4ba1407cda 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1624,7 +1624,7 @@ class binarytree:
binpkg_format = get_binpkg_format(
d.get("PATH"), remote=True
)
-except InvalidBinaryPackageFormat:
+except InvalidBinaryPackageFormat as e:
writemsg(
colorize(
"WARN",
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
commit: fe510e099bc9a8055c3ee50fced47fc3dc7ba166
Author: Zac Medico gentoo org>
AuthorDate: Fri Mar 1 17:09:56 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Mar 1 18:08:51 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fe510e09
doebuild: Call _setup_locale
Call _setup_locale in order to prevent an AssertionError from
config.environ() for the config phase (or any other phase for
that matter).
For returnproc or returnpid assume that the event loop is running
so we can't run the event loop to call _setup_locale in this case
and we have to assume the caller took care of it (otherwise
config.environ() will raise AssertionError).
Update DoebuildFdPipesTestCase to use EAPI 8 and test the
pkg_config function with an ebuild located in /var/db/pkg just
like emerge --config does. Set LC_ALL=C just before doebuild
calls in order to try and trigger the config.environ()
split_LC_ALL assertion.
Bug: https://bugs.gentoo.org/925863
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/doebuild.py | 8 +++
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 77 --
2 files changed, 52 insertions(+), 33 deletions(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index bc51fdff2d..942fa90101 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -43,6 +43,7 @@ portage.proxy.lazyimport.lazyimport(
"portage.util._async.SchedulerInterface:SchedulerInterface",
"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util.ExtractKernelVersion:ExtractKernelVersion",
+"_emerge.EbuildPhase:_setup_locale",
)
from portage import (
@@ -1034,6 +1035,13 @@ def doebuild(
myebuild, mydo, myroot, mysettings, debug, use_cache, mydbapi
)
+# For returnproc or returnpid assume that the event loop is running
+# so we can't run the event loop to call _setup_locale in this case
+# and we have to assume the caller took care of it (otherwise
+# config.environ() will raise AssertionError).
+if not (returnproc or returnpid):
+asyncio.run(_setup_locale(mysettings))
+
if mydo in clean_phases:
builddir_lock = None
if not returnpid and "PORTAGE_BUILDDIR_LOCKED" not in mysettings:
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index 678486ed16..b38605bb90 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Gentoo Authors
+# Copyright 2013-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import multiprocessing
@@ -27,20 +27,22 @@ class DoebuildFdPipesTestCase(TestCase):
output_fd = self.output_fd
ebuild_body = ["S=${WORKDIR}"]
-for phase_func in (
-"pkg_info",
-"pkg_nofetch",
-"pkg_pretend",
-"pkg_setup",
-"src_unpack",
-"src_prepare",
-"src_configure",
-"src_compile",
-"src_test",
-"src_install",
+for phase_func, default in (
+("pkg_info", False),
+("pkg_nofetch", False),
+("pkg_pretend", False),
+("pkg_setup", False),
+("pkg_config", False),
+("src_unpack", False),
+("src_prepare", True),
+("src_configure", False),
+("src_compile", False),
+("src_test", False),
+("src_install", False),
):
ebuild_body.append(
-("%s() { echo ${EBUILD_PHASE}" " 1>&%s; }") % (phase_func,
output_fd)
+("%s() { %secho ${EBUILD_PHASE}" " 1>&%s; }")
+% (phase_func, "default; " if default else "", output_fd)
)
ebuild_body.append("")
@@ -48,7 +50,7 @@ class DoebuildFdPipesTestCase(TestCase):
ebuilds = {
"app-misct/foo-1": {
-"EAPI": "5",
+"EAPI": "8",
"MISC_CONTENT": ebuild_body,
}
}
@@ -103,24 +105,33 @@ class DoebuildFdPipesTestCase(TestCase):
type_name="ebuild",
)
settings.setcpv(pkg)
-ebuildpath = portdb.findname(cpv)
-self.assertNotEqual(ebuildpath, None)
-
-for phase in (
-"info",
-"nofetch",
-"pretend",
-"setup",
-"unpack",
-"prepare",
-"configure",
-"compile",
-"test",
-"install",
-"qmerge",
-"clean",
-"merge",
+
+# Try to trigger the config.environ
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: d8089d1af39c80e1edfb1669ae92fef7ab30e08a
Author: Mike Gilbert gentoo org>
AuthorDate: Fri Mar 1 15:49:26 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 1 16:19:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8089d1a
Improve whitespace handling when parsing /proc/self/mountinfo
Only break lines on "\n" (line feed).
Only split lines on " " (space).
Bug: https://bugs.gentoo.org/925888
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/writeable_check.py | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/portage/util/writeable_check.py
b/lib/portage/util/writeable_check.py
index 3427315cf7..ad1d9edff0 100644
--- a/lib/portage/util/writeable_check.py
+++ b/lib/portage/util/writeable_check.py
@@ -47,6 +47,7 @@ def linux_ro_checker(dir_list):
"/proc/self/mountinfo",
encoding=_encodings["content"],
errors="replace",
+newline="\n",
) as f:
for line in f:
# we're interested in dir and both attr fields which always
@@ -58,7 +59,7 @@ def linux_ro_checker(dir_list):
# to the left of the ' - ', after the attr's, so split it there
mount = line.split(" - ", 1)
try:
-_dir, attr1 = mount[0].split()[4:6]
+_dir, attr1 = mount[0].split(" ")[4:6]
except ValueError:
# If it raises ValueError we can simply ignore the line.
invalids.append(line)
@@ -68,10 +69,10 @@ def linux_ro_checker(dir_list):
# for example: 16 1 0:16 / /root rw,noatime - lxfs rw
if len(mount) > 1:
try:
-attr2 = mount[1].split()[2]
+attr2 = mount[1].split(" ")[2]
except IndexError:
try:
-attr2 = mount[1].split()[1]
+attr2 = mount[1].split(" ")[1]
except IndexError:
invalids.append(line)
continue
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, src/
commit: a860484b6e8c8d107255f2105796ae7f58812e9f
Author: Mike Gilbert gentoo org>
AuthorDate: Thu Feb 29 19:38:20 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Thu Feb 29 19:38:20 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a860484b
Drop portage.util.libc extension module
This was originally added as a workaround for musl, where
ctypes.util.find_library fails.
Instead, we now try to load "libc.so" as a fallback and can just rely on
ctypes to call tolower() and toupper().
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/locale.py | 9 +++
src/meson.build| 9 ---
src/portage_util_libc.c| 67 --
3 files changed, 3 insertions(+), 82 deletions(-)
diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py
index d0edeb4afe..f45d761760 100644
--- a/lib/portage/util/locale.py
+++ b/lib/portage/util/locale.py
@@ -43,12 +43,9 @@ def _check_locale(silent):
"""
The inner locale check function.
"""
-try:
-from portage.util import libc
-except ImportError:
-(libc, _) = load_libc()
-if libc is None:
-return None
+(libc, _) = load_libc()
+if libc is None:
+return None
lc = list(range(ord("a"), ord("z") + 1))
uc = list(range(ord("A"), ord("Z") + 1))
diff --git a/src/meson.build b/src/meson.build
index cbc7aa611c..6a36724ceb 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -2,14 +2,6 @@
# and for development. Meson does not allow you to build in-place and Python
# cannot create a single namespace from two identically-named paths.
-libc_ext = py.extension_module(
-'libc',
-'portage_util_libc.c',
-dependencies : py.dependency(),
-subdir : 'portage' / 'util',
-install : true
-)
-
whirlpool_ext = py.extension_module(
'_whirlpool',
'portage_util__whirlpool.c',
@@ -21,7 +13,6 @@ whirlpool_ext = py.extension_module(
run_command(
[
'ln', '-srnf',
-libc_ext.full_path(),
whirlpool_ext.full_path(),
meson.project_source_root() / 'lib' / 'portage' / 'util/'
],
diff --git a/src/portage_util_libc.c b/src/portage_util_libc.c
deleted file mode 100644
index 12b2440c72..00
--- a/src/portage_util_libc.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/* Copyright 2005-2020 Gentoo Authors
- * Distributed under the terms of the GNU General Public License v2
- */
-
-#include
-#include
-#include
-
-static PyObject * _libc_tolower(PyObject *, PyObject *);
-static PyObject * _libc_toupper(PyObject *, PyObject *);
-
-static PyMethodDef LibcMethods[] = {
- {
- .ml_name = "tolower",
- .ml_meth = _libc_tolower,
- .ml_flags = METH_VARARGS,
- .ml_doc = "Convert to lower case using system locale."
-
- },
- {
- .ml_name = "toupper",
- .ml_meth = _libc_toupper,
- .ml_flags = METH_VARARGS,
- .ml_doc = "Convert to upper case using system locale."
- },
- {NULL, NULL, 0, NULL}
-};
-
-static struct PyModuleDef moduledef = {
- PyModuleDef_HEAD_INIT,
- .m_name = "libc",
- .m_doc = "Module for converting case using the system locale",
- .m_size = -1,
- .m_methods = LibcMethods,
-};
-
-PyMODINIT_FUNC
-PyInit_libc(void)
-{
- PyObject *m;
- m = PyModule_Create(&moduledef);
- return m;
-}
-
-
-static PyObject *
-_libc_tolower(PyObject *self, PyObject *args)
-{
- int c;
-
- if (!PyArg_ParseTuple(args, "i", &c))
- return NULL;
-
- return Py_BuildValue("i", tolower(c));
-}
-
-
-static PyObject *
-_libc_toupper(PyObject *self, PyObject *args)
-{
- int c;
-
- if (!PyArg_ParseTuple(args, "i", &c))
- return NULL;
-
- return Py_BuildValue("i", toupper(c));
-}
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/, ...
commit: b2d8226af4589d95f44d20d441056f645a523039
Author: Zac Medico gentoo org>
AuthorDate: Thu Feb 29 04:26:02 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Feb 29 04:37:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2d8226a
Delete compat_coroutine module
The compat_coroutine module has been unused since the migration
to PEP 492 async and await syntax in 2021, which began in commit
b3b9acc13c43 and was completed in commit bcda30d0a6fa.
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/futures/meson.build | 1 -
.../tests/util/futures/test_compat_coroutine.py| 210 -
lib/portage/util/futures/_asyncio/__init__.py | 15 +-
lib/portage/util/futures/compat_coroutine.py | 142 --
lib/portage/util/futures/meson.build | 1 -
5 files changed, 1 insertion(+), 368 deletions(-)
diff --git a/lib/portage/tests/util/futures/meson.build
b/lib/portage/tests/util/futures/meson.build
index 877acc27cd..cb78314844 100644
--- a/lib/portage/tests/util/futures/meson.build
+++ b/lib/portage/tests/util/futures/meson.build
@@ -1,6 +1,5 @@
py.install_sources(
[
-'test_compat_coroutine.py',
'test_done_callback.py',
'test_done_callback_after_exit.py',
'test_iter_completed.py',
diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py
b/lib/portage/tests/util/futures/test_compat_coroutine.py
deleted file mode 100644
index b25708886c..00
--- a/lib/portage/tests/util/futures/test_compat_coroutine.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Copyright 2018 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import (
-coroutine,
-coroutine_return,
-)
-from portage.util.futures._sync_decorator import _sync_decorator, _sync_methods
-from portage.tests import TestCase
-
-
-class CompatCoroutineTestCase(TestCase):
-def test_returning_coroutine(self):
-@coroutine
-def returning_coroutine(loop=None):
-yield asyncio.sleep(0, loop=loop)
-coroutine_return("success")
-
-loop = asyncio.get_event_loop()
-self.assertEqual(
-"success",
-
asyncio.get_event_loop().run_until_complete(returning_coroutine(loop=loop)),
-)
-
-def test_raising_coroutine(self):
-class TestException(Exception):
-pass
-
-@coroutine
-def raising_coroutine(loop=None):
-yield asyncio.sleep(0, loop=loop)
-raise TestException("exception")
-
-loop = asyncio.get_event_loop()
-self.assertRaises(
-TestException, loop.run_until_complete,
raising_coroutine(loop=loop)
-)
-
-def test_catching_coroutine(self):
-class TestException(Exception):
-pass
-
-@coroutine
-def catching_coroutine(loop=None):
-loop = asyncio._wrap_loop(loop)
-future = loop.create_future()
-loop.call_soon(future.set_exception, TestException("exception"))
-try:
-yield future
-except TestException:
-self.assertTrue(True)
-else:
-self.assertTrue(False)
-coroutine_return("success")
-
-loop = asyncio.get_event_loop()
-self.assertEqual(
-"success", loop.run_until_complete(catching_coroutine(loop=loop))
-)
-
-def test_cancelled_coroutine(self):
-"""
-Verify that a coroutine can handle (and reraise) asyncio.CancelledError
-in order to perform any necessary cleanup. Note that the
-asyncio.CancelledError will only be thrown in the coroutine if there's
-an opportunity (yield) before the generator raises StopIteration.
-"""
-loop = asyncio.get_event_loop()
-ready_for_exception = loop.create_future()
-exception_in_coroutine = loop.create_future()
-
-@coroutine
-def cancelled_coroutine(loop=None):
-loop = asyncio._wrap_loop(loop)
-while True:
-task = loop.create_future()
-try:
-ready_for_exception.set_result(None)
-yield task
-except BaseException as e:
-# Since python3.8, asyncio.CancelledError inherits
-# from BaseException.
-task.done() or task.cancel()
-exception_in_coroutine.set_exception(e)
-raise
-else:
-exception_in_coroutine.set_result(None)
-
-future = cancelled_coroutine(loop=loop)
-loop.run_until_complete(ready_for_exception)
-future.cancel()
-
-self.assertRaises(asyncio.CancelledError, loop.run_until_complete,
future)
-
-sel
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: fa7fb4c5119aa077a0731e1f8892ea0791a4968b
Author: Sam James gentoo org>
AuthorDate: Wed Feb 28 16:00:57 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 28 16:01:04 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa7fb4c5
gpkg: placate black
Fixes: 957902f84edece635210689f46e20741e76d9dba
Signed-off-by: Sam James gentoo.org>
lib/portage/gpkg.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 5cd1f2394e..2b957d58c4 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -656,7 +656,9 @@ class tar_safe_extract:
raise ValueError("Path traversal detected.")
if member.isdev():
writemsg(
-colorize("BAD", f"Danger: device file detected:
{member.name}\n")
+colorize(
+"BAD", f"Danger: device file detected:
{member.name}\n"
+)
)
raise ValueError("Device file detected.")
if member.islnk() and (member.linkname not in self.file_list):
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 957902f84edece635210689f46e20741e76d9dba
Author: Sam James gentoo org>
AuthorDate: Wed Feb 28 15:51:15 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 28 15:51:15 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=957902f8
gpkg: add missing new lines to error messages
Signed-off-by: Sam James gentoo.org>
lib/portage/gpkg.py | 18 +-
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index edb0e43fbf..5cd1f2394e 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -382,7 +382,7 @@ class tar_stream_reader:
try:
if self.proc.wait() != os.EX_OK:
if not self.killed:
-writemsg(colorize("BAD", f"GPKG external program
failed."))
+writemsg(colorize("BAD", f"GPKG external program
failed.\n"))
raise CompressorOperationFailed("decompression failed")
finally:
self.proc.stdout.close()
@@ -418,7 +418,7 @@ class checksum_helper:
else:
self.uid = pwd.getpwnam(drop_user).pw_uid
except KeyError:
-writemsg(colorize("BAD", f"!!! Failed to find user
{drop_user}."))
+writemsg(colorize("BAD", f"!!! Failed to find user
{drop_user}.\n"))
raise
try:
@@ -428,7 +428,7 @@ class checksum_helper:
else:
self.gid = grp.getgrnam(drop_group).gr_gid
except KeyError:
-writemsg(colorize("BAD", f"!!! Failed to find group
{drop_group}."))
+writemsg(colorize("BAD", f"!!! Failed to find group
{drop_group}.\n"))
raise
else:
self.uid = None
@@ -636,33 +636,33 @@ class tar_safe_extract:
):
writemsg(
colorize(
-"BAD", f"Danger: duplicate files detected:
{member.name}"
+"BAD", f"Danger: duplicate files detected:
{member.name}\n"
)
)
raise ValueError("Duplicate files detected.")
if member.name.startswith("/"):
writemsg(
colorize(
-"BAD", f"Danger: absolute path detected:
{member.name}"
+"BAD", f"Danger: absolute path detected:
{member.name}\n"
)
)
raise ValueError("Absolute path detected.")
if member.name.startswith("../") or ("/../" in member.name):
writemsg(
colorize(
-"BAD", f"Danger: path traversal detected:
{member.name}"
+"BAD", f"Danger: path traversal detected:
{member.name}\n"
)
)
raise ValueError("Path traversal detected.")
if member.isdev():
writemsg(
-colorize("BAD", f"Danger: device file detected:
{member.name}")
+colorize("BAD", f"Danger: device file detected:
{member.name}\n")
)
raise ValueError("Device file detected.")
if member.islnk() and (member.linkname not in self.file_list):
writemsg(
colorize(
-"BAD", f"Danger: hardlink escape detected:
{member.name}"
+"BAD", f"Danger: hardlink escape detected:
{member.name}\n"
)
)
raise ValueError("Hardlink escape detected.")
@@ -995,7 +995,7 @@ class gpkg:
image_safe.extractall(decompress_dir)
image_tar.close()
except Exception as ex:
-writemsg(colorize("BAD", "!!!Extract failed."))
+writemsg(colorize("BAD", "!!!Extract failed.\n"))
raise
finally:
if not image_tar.closed:
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: f070275fe05d5053c3756ebb5d0a602db8ba515d
Author: Sam James gentoo org>
AuthorDate: Wed Feb 28 15:48:48 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 28 15:49:16 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f070275f
binpkg: add another missing newline to error message
```
Error reading binpkg
'/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar': [Errno 22]
Invalid argument!!! Invalid binary package:
'/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar', Error reading
binpkg '/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar': [Errno
22] Invalid argument
```
Bug: https://bugs.gentoo.org/925714
Signed-off-by: Sam James gentoo.org>
lib/portage/binpkg.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
index 9ecd52cf3c..a48e09bdb8 100644
--- a/lib/portage/binpkg.py
+++ b/lib/portage/binpkg.py
@@ -54,7 +54,7 @@ def get_binpkg_format(binpkg_path, check_file=False,
remote=False):
# We got many different exceptions here, so have to catch all of them.
file_format = None
writemsg(
-colorize("ERR", f"Error reading binpkg '{binpkg_path}': {err}"),
+colorize("ERR", f"Error reading binpkg '{binpkg_path}': {err}\n"),
)
raise InvalidBinaryPackageFormat(f"Error reading binpkg
'{binpkg_path}': {err}")
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/
commit: e882b1e956d50808a0143875a8ca35f2fc21f368
Author: Zac Medico gentoo org>
AuthorDate: Wed Feb 28 06:25:45 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 28 06:25:45 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e882b1e9
UnshareNetTestCase: Initialize ABILITY_TO_UNSHARE in setUp
Initialize ABILITY_TO_UNSHARE in setUp so that _unshare_validate
uses the correct PORTAGE_MULTIPROCESSING_START_METHOD setup
from super().setUp(), eliminating messages like this from CI
runs for the multiprocessing start method spawn:
/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/popen_fork.py:66:
DeprecationWarning: This process (pid=2886) is multi-threaded, use of
fork() may lead to deadlocks in the child.
The cause of these messages can be traced by patching python's
multiprocessing popen_fork.py like this:
--- /usr/lib/python3.12/multiprocessing/popen_fork.py
+++ /usr/lib/python3.12/multiprocessing/popen_fork.py
@@ -2,2 +2,3 @@
import signal
+import traceback
@@ -62,2 +63,3 @@
def _launch(self, process_obj):
+traceback.print_stack()
code = 1
Fixes: 3110ec376cbc ("actions: Fix interaction between start-method and
pytest-xdist")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/process/test_unshare_net.py | 18 +-
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/lib/portage/tests/process/test_unshare_net.py
b/lib/portage/tests/process/test_unshare_net.py
index dabf15585f..ad3b288ef4 100644
--- a/lib/portage/tests/process/test_unshare_net.py
+++ b/lib/portage/tests/process/test_unshare_net.py
@@ -20,19 +20,27 @@ ping -c 1 -W 1 10.0.0.1 || exit 1
ping -c 1 -W 1 ::1 || exit 1
ping -c 1 -W 1 fd::1 || exit 1
"""
-ABILITY_TO_UNSHARE = portage.process._unshare_validate(CLONE_NEWNET)
class UnshareNetTestCase(TestCase):
-@pytest.mark.skipif(
-ABILITY_TO_UNSHARE != 0,
-reason=f"Unable to unshare: {errno.errorcode.get(ABILITY_TO_UNSHARE,
'?')}",
-)
+def setUp(self):
+"""
+Initialize ABILITY_TO_UNSHARE in setUp so that _unshare_validate
+uses the correct PORTAGE_MULTIPROCESSING_START_METHOD setup
+from super().setUp().
+"""
+super().setUp()
+self.ABILITY_TO_UNSHARE =
portage.process._unshare_validate(CLONE_NEWNET)
+
@pytest.mark.skipif(
portage.process.find_binary("ping") is None, reason="ping not found"
)
@pytest.mark.skipif(platform.system() != "Linux", reason="not Linux")
def testUnshareNet(self):
+if self.ABILITY_TO_UNSHARE != 0:
+pytest.skip(
+f"Unable to unshare:
{errno.errorcode.get(self.ABILITY_TO_UNSHARE, '?')}"
+)
env = os.environ.copy()
env["IPV6"] = "1" if portage.process._has_ipv6() else ""
self.assertEqual(
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dbapi/
commit: e9bdb7342b3048ab3236bff9d94ce733bb877d8e
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 27 04:02:28 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 27 04:03:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e9bdb734
BinarytreeTestCase: Use temporary TMPDIR
Create a temporary TMPDIR which prevents test methods of this
class from leaving behind an empty /tmp/Packages file if TMPDIR
is initially unset.
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/dbapi/test_bintree.py | 22 +-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/lib/portage/tests/dbapi/test_bintree.py
b/lib/portage/tests/dbapi/test_bintree.py
index 018f1cf9bd..91ac338a05 100644
--- a/lib/portage/tests/dbapi/test_bintree.py
+++ b/lib/portage/tests/dbapi/test_bintree.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Gentoo Authors
+# Copyright 2022-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from unittest.mock import MagicMock, patch, call
@@ -13,6 +13,26 @@ from portage.const import BINREPOS_CONF_FILE
class BinarytreeTestCase(TestCase):
+@classmethod
+def setUpClass(cls):
+"""
+Create a temporary TMPDIR which prevents test
+methods of this class from leaving behind an empty
+/tmp/Packages file if TMPDIR is initially unset.
+"""
+cls._orig_tmpdir = os.environ.get("TMPDIR")
+cls._tmpdir = tempfile.TemporaryDirectory()
+os.environ["TMPDIR"] = cls._tmpdir.name
+
+@classmethod
+def tearDownClass(cls):
+cls._tmpdir.cleanup()
+if cls._orig_tmpdir is None:
+os.environ.pop("TMPDIR", None)
+else:
+os.environ["TMPDIR"] = cls._orig_tmpdir
+del cls._orig_tmpdir, cls._tmpdir
+
def test_required_init_params(self):
with self.assertRaises(TypeError) as cm:
binarytree()
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/resolver/, cnf/sets/, lib/portage/tests/sets/base/
commit: 1d3e3843f2a51c581d344540c5c6ee266afa30d2
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 25 22:57:43 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 26 23:17:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1d3e3843
cnf: sets: Migrate @golang-rebuild to dev-lang/go
Bug: https://bugs.gentoo.org/919751
Signed-off-by: Zac Medico gentoo.org>
cnf/sets/portage.conf| 6 --
lib/portage/tests/resolver/ResolverPlayground.py | 10 +-
lib/portage/tests/sets/base/test_variable_set.py | 8 ++--
3 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf
index 2e02f91f97..c272f98db1 100644
--- a/cnf/sets/portage.conf
+++ b/cnf/sets/portage.conf
@@ -110,12 +110,6 @@ class = portage.sets.dbapi.UnavailableBinaries
[changed-deps]
class = portage.sets.dbapi.ChangedDepsSet
-# Installed packages for which vdb *DEPEND includes dev-lang/go.
-[golang-rebuild]
-class = portage.sets.dbapi.VariableSet
-variable = BDEPEND
-includes = dev-lang/go
-
# Installed packages for which vdb *DEPEND includes virtual/rust
[rust-rebuild]
class = portage.sets.dbapi.VariableSet
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index 75c86b615c..c0455415a1 100644
--- a/lib/portage/tests/resolver/ResolverPlayground.py
+++ b/lib/portage/tests/resolver/ResolverPlayground.py
@@ -28,7 +28,7 @@ from portage.exception import InvalidBinaryPackageFormat
from portage.gpg import GPG
import _emerge
-from _emerge.actions import _calc_depclean
+from _emerge.actions import _calc_depclean, expand_set_arguments
from _emerge.Blocker import Blocker
from _emerge.create_depgraph_params import create_depgraph_params
from _emerge.DependencyArg import DependencyArg
@@ -747,6 +747,14 @@ class ResolverPlayground:
self.settings, self.trees, options, params, None
)
+atoms, retval = expand_set_arguments(
+atoms, action, self.trees[self.eroot]["root_config"]
+)
+if retval != os.EX_OK:
+raise AssertionError(
+f"expand_set_arguments failed with retval {retval}"
+)
+
if params_action == "remove":
depclean_result = _calc_depclean(
self.settings,
diff --git a/lib/portage/tests/sets/base/test_variable_set.py
b/lib/portage/tests/sets/base/test_variable_set.py
index 9e90ee6dd7..60c43a5b83 100644
--- a/lib/portage/tests/sets/base/test_variable_set.py
+++ b/lib/portage/tests/sets/base/test_variable_set.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Gentoo Authors
+# Copyright 2022-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -10,6 +10,10 @@ from portage.tests.resolver.ResolverPlayground import (
class VariableSetTestCase(TestCase):
def testVariableSetEmerge(self):
+
+# Using local set definition because @golang-rebuild migrated to
dev-lang/go since bug 919751.
+golang_rebuild =
"{class=portage.sets.dbapi.VariableSet,variable=BDEPEND,includes=dev-lang/go}"
+
ebuilds = {
"dev-go/go-pkg-1": {"BDEPEND": "dev-lang/go"},
"www-client/firefox-1": {
@@ -21,7 +25,7 @@ class VariableSetTestCase(TestCase):
test_cases = (
ResolverPlaygroundTestCase(
-["@golang-rebuild"],
+[f"@golang-rebuild{golang_rebuild}"],
mergelist=["dev-go/go-pkg-1"],
success=True,
),
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/, lib/_emerge/
commit: 0ff7a3b28e0ec63d68d32e01145db8962d53774d
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 26 05:09:21 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 26 05:09:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0ff7a3b2
SpawnProcess: Wait for async set_term_size
Use the SpawnProcess _unregister method to handle async
set_term_size results, avoiding possible messages like
bug 925456 triggered:
[ERROR] Task was destroyed but it is pending!
Also update _BinpkgFetcherProcess and _EbuildFetcherProcess
which inherit the _pty_ready attribute from SpawnProcess.
Fixes: f97e414ce980 ("set_term_size: Wait asynchronously if event loop is
running")
Bug: https://bugs.gentoo.org/923750
Bug: https://bugs.gentoo.org/925456
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/BinpkgFetcher.py | 6 --
lib/_emerge/EbuildFetcher.py | 6 --
lib/_emerge/SpawnProcess.py | 14 +-
lib/portage/output.py| 13 +
lib/portage/util/_pty.py | 23 ---
5 files changed, 46 insertions(+), 16 deletions(-)
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index a1524dc009..587e4a57a3 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.AsynchronousLock import AsynchronousLock
@@ -233,7 +233,9 @@ class _BinpkgFetcherProcess(SpawnProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def sync_timestamp(self):
diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py
index 7a45d95172..81d4b1054b 100644
--- a/lib/_emerge/EbuildFetcher.py
+++ b/lib/_emerge/EbuildFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -394,7 +394,9 @@ class _EbuildFetcherProcess(ForkProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def _eerror(self, lines):
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index b63afae01c..9fc12c42e5 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -46,6 +46,7 @@ class SpawnProcess(SubProcess):
+ (
"_main_task",
"_main_task_cancel",
+"_pty_ready",
"_selinux_type",
)
)
@@ -193,6 +194,9 @@ class SpawnProcess(SubProcess):
self._main_task.add_done_callback(self._main_exit)
async def _main(self, build_logger, pipe_logger, loop=None):
+if isinstance(self._pty_ready, asyncio.Future):
+await self._pty_ready
+self._pty_ready = None
try:
if pipe_logger.poll() is None:
await pipe_logger.async_wait()
@@ -238,7 +242,9 @@ class SpawnProcess(SubProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def _spawn(
@@ -258,6 +264,12 @@ class SpawnProcess(SubProcess):
SubProcess._unregister(self)
if self._main_task is not None:
self._main_task.done() or self._main_task.cancel()
+if isinstance(self._pty_ready, asyncio.Future):
+(
+self._pty_ready.done()
+and (self._pty_ready.cancelled() or self._pty_ready.result()
or True)
+) or self._pty_ready.cancel()
+self._pty_ready = None
def _cancel(self):
if self._main_task is not None:
diff --git a/lib/portage/output.py b/lib/portage/output.py
index 7d3a6278f3..4408705c45 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -8,6 +8,7 @@ import itertools
import re
import subprocess
import sys
+from typing import Optional
import portage
@@ -554,10 +555,16 @@ def get_term_size(fd=None):
return (0, 0)
-def set_term_size(lines, columns, fd):
+def set_term_size(lines: int, columns: int, fd: int) ->
Optional[asyncio.Future]:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/, /, lib/portage/
commit: 19e27e0415fd321c39104f7d687bcdc4f4132e24
Author: Mike Gilbert gentoo org>
AuthorDate: Sun Feb 25 18:10:15 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Mon Feb 26 04:10:32 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=19e27e04
Add workaround for loading libc on musl
musl libc has no soname, which causes ctypes.util.find_library to fail.
As a fallback, try to load "libc.so".
Signed-off-by: Mike Gilbert gentoo.org>
NEWS| 7 +++
lib/portage/dbapi/_MergeProcess.py | 4 ++--
lib/portage/dbapi/_SyncfsProcess.py | 14 --
lib/portage/process.py | 16 +---
lib/portage/util/_ctypes.py | 15 +++
lib/portage/util/locale.py | 7 ++-
6 files changed, 35 insertions(+), 28 deletions(-)
diff --git a/NEWS b/NEWS
index eb84651b53..258d800373 100644
--- a/NEWS
+++ b/NEWS
@@ -6,8 +6,15 @@ Release notes take the form of the following optional
categories:
* Bug fixes
* Cleanups
+portage-3.0.64 (UNRELEASED)
+--
+
+Bug fixes:
+
+
portage-3.0.63 (2024-02-25)
--
+* ctypes: Add workaround for loading libc on musl
Bug fixes:
* emerge: Skip installed packages with emptytree in depgraph selection (bug
#651018).
diff --git a/lib/portage/dbapi/_MergeProcess.py
b/lib/portage/dbapi/_MergeProcess.py
index dd5ad71cf8..d9ab2b47aa 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -10,7 +10,7 @@ import fcntl
import portage
from portage import os, _unicode_decode
from portage.package.ebuild._ipc.QueryCommand import QueryCommand
-from portage.util._ctypes import find_library
+from portage.util._ctypes import load_libc
import portage.elog.messages
from portage.util._async.ForkProcess import ForkProcess
from portage.util import no_color
@@ -64,7 +64,7 @@ class MergeProcess(ForkProcess):
# process, so that it's only done once rather than
# for each child process.
if platform.system() == "Linux" and "merge-sync" in settings.features:
-find_library("c")
+load_libc()
# Inherit stdin by default, so that the pdb SIGUSR1
# handler is usable for the subprocess.
diff --git a/lib/portage/dbapi/_SyncfsProcess.py
b/lib/portage/dbapi/_SyncfsProcess.py
index ddc2240071..300ae53985 100644
--- a/lib/portage/dbapi/_SyncfsProcess.py
+++ b/lib/portage/dbapi/_SyncfsProcess.py
@@ -4,7 +4,7 @@
import functools
from portage import os
-from portage.util._ctypes import find_library, LoadLibrary
+from portage.util._ctypes import load_libc
from portage.util._async.ForkProcess import ForkProcess
@@ -24,15 +24,9 @@ class SyncfsProcess(ForkProcess):
@staticmethod
def _get_syncfs():
-filename = find_library("c")
-if filename is not None:
-library = LoadLibrary(filename)
-if library is not None:
-try:
-return library.syncfs
-except AttributeError:
-pass
-
+(libc, _) = load_libc()
+if libc is not None:
+return getattr(libc, "syncfs", None)
return None
@staticmethod
diff --git a/lib/portage/process.py b/lib/portage/process.py
index cc9ed7bf78..b652e32942 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -37,7 +37,7 @@ portage.proxy.lazyimport.lazyimport(
from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
from portage.exception import CommandNotFound
from portage.proxy.objectproxy import ObjectProxy
-from portage.util._ctypes import find_library, LoadLibrary, ctypes
+from portage.util._ctypes import load_libc, LoadLibrary, ctypes
try:
from portage.util.netlink import RtNetlink
@@ -960,11 +960,9 @@ def _exec(
have_unshare = False
libc = None
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
-filename = find_library("c")
-if filename is not None:
-libc = LoadLibrary(filename)
-if libc is not None:
-have_unshare = hasattr(libc, "unshare")
+(libc, _) = load_libc()
+if libc is not None:
+have_unshare = hasattr(libc, "unshare")
if not have_unshare:
# unshare() may not be supported by libc
@@ -1212,11 +1210,7 @@ class _unshare_validator:
"""
# This ctypes library lookup caches the result for use in the
# subprocess when the multiprocessing start method is fork.
-filename = find_library("c")
-if filename is None:
-return errno.ENOTSUP
-
-libc = LoadLibrary(filename)
+(libc, filename) = load_libc()
if libc is None:
return errno.ENOTSUP
diff --git a/lib/portage/util/_ctypes.py b/lib/portage/util/_ctypes.py
index e6d1e327cb..04e965ba92 100644
--- a/lib/portage/util/_ctypes.py
+++ b/lib/portage/util/_ctypes.py
@@ -48,3
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 0855821572f32e81b031a250f7491f34a2fd4078
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 24 23:29:29 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Feb 25 08:24:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=08558215
dbapi: Fix TypeError when passing Exception to warnings.warn
If an Exception is passed as a message to warnings.warn then
it triggers this TypeError:
if metadata_updates:
try:
aux_update(cpv, metadata_updates)
except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
> warnings.warn(e)
E TypeError: expected string or bytes-like object, got
'CorruptionKeyError'
Fixes: fb1d0a22f657 ("dbapi: KeyError tolerance during package moves")
Bug: https://bugs.gentoo.org/922935
Signed-off-by: Zac Medico gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1282
Signed-off-by: Sam James gentoo.org>
lib/portage/dbapi/__init__.py | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 6f95b93a21..9105227c77 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -1,11 +1,12 @@
-# Copyright 1998-2023 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["dbapi"]
import functools
+import logging
import re
-import warnings
+import sys
from typing import Any, Dict, List, Optional, Tuple
from collections.abc import Sequence
@@ -429,7 +430,9 @@ class dbapi:
try:
aux_update(cpv, metadata_updates)
except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
-warnings.warn(e)
+logging.warning(
+f"{e.__class__.__name__}: {e}", exc_info=sys.exc_info()
+)
if onUpdate:
onUpdate(maxval, i + 1)
if onProgress:
@@ -477,6 +480,6 @@ class dbapi:
try:
self.aux_update(mycpv, mydata)
except CorruptionKeyError as e:
-warnings.warn(e)
+logging.warning(f"{e.__class__.__name__}: {e}",
exc_info=sys.exc_info())
continue
return moves
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/emerge/
commit: 8a2f1d14788d107ec54dc53c9ef1cf00ee310d51 Author: Gábor Oszkár Dénes protonmail com> AuthorDate: Sat Feb 24 20:48:05 2024 + Commit: Sam James gentoo org> CommitDate: Sun Feb 25 08:25:06 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8a2f1d14 test_baseline: Improve robustness with cleanup The baseline tests need to cleanup the ResolverPlayground after each testcase, with each different parametrization. This is ensured by making the scope of the playground fixture the function instead of the module. With module the cleanup only happens before/after the switch from/to xpak and gpkg. Signed-off-by: Gábor Oszkár Dénes protonmail.com> Closes: https://github.com/gentoo/portage/pull/1281 Signed-off-by: Sam James gentoo.org> lib/portage/tests/emerge/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/tests/emerge/conftest.py b/lib/portage/tests/emerge/conftest.py index 580d1e09ab..356e09879c 100644 --- a/lib/portage/tests/emerge/conftest.py +++ b/lib/portage/tests/emerge/conftest.py @@ -406,7 +406,7 @@ def async_loop(): yield asyncio._wrap_loop() -@pytest.fixture(params=SUPPORTED_GENTOO_BINPKG_FORMATS, scope="module") +@pytest.fixture(params=SUPPORTED_GENTOO_BINPKG_FORMATS, scope="function") def playground(request, tmp_path_factory): """Fixture that provides instances of ``ResolverPlayground`` each one with one supported value for ``BINPKG_FORMAT``."""
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 92615cd37d7a1efce923c474e455f59fe61a589c
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 25 05:09:48 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Feb 25 08:24:59 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=92615cd3
_start_proc: Prevent premature ForkProcess garbage collection
In order to prevent the constructed ForkProcess instance from
being prematurely garbage collected, return a reference to the
caller inside of a _GCProtector object, preventing messages
reported in bug 925456 like this:
[ERROR] Task was destroyed but it is pending!
Fixes: a69c1b853a47 ("process.spawn: Use multiprocessing.Process for
returnproc")
Bug: https://bugs.gentoo.org/925456
Signed-off-by: Zac Medico gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1284
Signed-off-by: Sam James gentoo.org>
lib/portage/process.py | 41 +++--
1 file changed, 39 insertions(+), 2 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index d16262e75a..cc9ed7bf78 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -36,6 +36,7 @@ portage.proxy.lazyimport.lazyimport(
from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
from portage.exception import CommandNotFound
+from portage.proxy.objectproxy import ObjectProxy
from portage.util._ctypes import find_library, LoadLibrary, ctypes
try:
@@ -1493,8 +1494,44 @@ def _start_proc(
proc.start()
# ForkProcess conveniently holds a MultiprocessingProcess
-# instance that is suitable to return here.
-return proc._proc
+# instance that is suitable to return here, but use _GCProtector
+# to protect the ForkProcess instance from being garbage collected
+# and triggering messages like this (bug 925456):
+# [ERROR] Task was destroyed but it is pending!
+return _GCProtector(proc._proc, proc.async_wait)
+
+
+class _GCProtector(ObjectProxy):
+"""
+Proxy a target object, and also hold a reference to something
+extra in order to protect it from garbage collection. Override
+the wait method to first call target's wait method and then
+wait for extra (a coroutine function) before returning the result.
+"""
+
+__slots__ = ("_extra", "_target")
+
+def __init__(self, target, extra):
+super().__init__()
+object.__setattr__(self, "_target", target)
+object.__setattr__(self, "_extra", extra)
+
+def _get_target(self):
+return object.__getattribute__(self, "_target")
+
+def __getattribute__(self, attr):
+if attr == "wait":
+return object.__getattribute__(self, attr)
+return getattr(object.__getattribute__(self, "_target"), attr)
+
+async def wait(self):
+"""
+Wrap the target's wait method to also wait for an extra
+coroutine function.
+"""
+result = await object.__getattribute__(self, "_target").wait()
+await object.__getattribute__(self, "_extra")()
+return result
def find_binary(binary):
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/
commit: 9e84ef57ba747766c9147c1ac1b247faa1f05956
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 25 00:31:13 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 25 00:42:46 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9e84ef57
testSpawnReturnProcTerminate: Fix integer in spawn command argument
The invalid integer in the spawn command argument intermittently
triggered this error when SIGTERM did not arrive until after the
exec call:
- Captured stderr call -
Process ForkProcess-23:
Traceback (most recent call last):
File "/home/runner/work/portage/portage/lib/portage/process.py", line 818, in
_exec_wrapper
_exec(
File "/home/runner/work/portage/portage/lib/portage/process.py", line 1068,
in _exec
_exec2(
File "/home/runner/work/portage/portage/lib/portage/process.py", line 1166,
in _exec2
os.execve(binary, myargs, env)
TypeError: expected str, bytes or os.PathLike object, not int
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File
"/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/process.py",
line 314, in _bootstrap
self.run()
File
"/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/process.py",
line 108, in run
self._target(*self._args, **self._kwargs)
File
"/home/runner/work/portage/portage/lib/portage/util/_async/ForkProcess.py",
line 328, in _bootstrap
sys.exit(target(*(args or []), **(kwargs or {})))
^^^
File "/home/runner/work/portage/portage/lib/portage/process.py", line 1441,
in __call__
return self._target(*args, **kwargs)
^
File "/home/runner/work/portage/portage/lib/portage/process.py", line 853, in
_exec_wrapper
writemsg(f"{e}:\n {' '.join(mycommand)}\n", noiselevel=-1)
^^^
TypeError: sequence item 1: expected str instance, int found
Bug: https://bugs.gentoo.org/916566#c20
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/process/test_spawn_returnproc.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/tests/process/test_spawn_returnproc.py
b/lib/portage/tests/process/test_spawn_returnproc.py
index 6d823d9c3d..8fbf54d0d2 100644
--- a/lib/portage/tests/process/test_spawn_returnproc.py
+++ b/lib/portage/tests/process/test_spawn_returnproc.py
@@ -32,7 +32,7 @@ class SpawnReturnProcTestCase(TestCase):
loop = global_event_loop()
async def watch_pid():
-proc = spawn([sleep_binary, ], returnproc=True)
+proc = spawn([sleep_binary, ""], returnproc=True)
proc.terminate()
self.assertEqual(await proc.wait(), -signal.SIGTERM)
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 3f4250dc7d32e9915224b1c9c4bc04c2740abcda
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 23 20:35:04 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 24 20:07:39 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3f4250dc
process.spawn: Fix logic for missing libc.unshare on musl
Fix unshare_* variables to be False when the libc is missing,
libc.unshare is missing, or libc.unshare fails.
Also, if socket.sethostname is missing then _exec2 needs
libc for the network-sandbox sethostname call which is wrapped
by a blanket Exception handler.
Fixes: 419cce79f908 ("process._exec: Use _start_fork for os.fork() error
handling")
Bug: https://bugs.gentoo.org/925311
Signed-off-by: Zac Medico gentoo.org>
lib/portage/process.py | 213 +
1 file changed, 110 insertions(+), 103 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index f4758c824c..d16262e75a 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -956,114 +956,119 @@ def _exec(
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
# Unshare (while still uid==0)
+have_unshare = False
+libc = None
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
-# unshare() may not be supported by libc
-if not hasattr(libc, "unshare"):
-unshare_net = False
-unshare_ipc = False
-unshare_mount = False
-unshare_pid = False
-else:
-# Since a failed unshare call could corrupt process
-# state, first validate that the call can succeed.
-# The parent process should call _unshare_validate
-# before it forks, so that all child processes can
-# reuse _unshare_validate results that have been
-# cached by the parent process.
-errno_value = _unshare_validate(unshare_flags)
-if errno_value == 0 and libc.unshare(unshare_flags) != 0:
-errno_value = ctypes.get_errno()
-if errno_value != 0:
-involved_features = []
-if unshare_ipc:
-involved_features.append("ipc-sandbox")
-if unshare_mount:
-involved_features.append("mount-sandbox")
-if unshare_net:
-involved_features.append("network-sandbox")
-if unshare_pid:
-involved_features.append("pid-sandbox")
-
-writemsg(
-'Unable to unshare: %s (for FEATURES="%s")\n'
-% (
-errno.errorcode.get(errno_value, "?"),
-" ".join(involved_features),
-),
-noiselevel=-1,
-)
-else:
-if unshare_pid:
-# pid namespace requires us to become init
-binary, myargs = (
-portage._python_interpreter,
-[
-portage._python_interpreter,
-os.path.join(portage._bin_path,
"pid-ns-init"),
-_unicode_encode("" if uid is None else
str(uid)),
-_unicode_encode("" if gid is None else
str(gid)),
-_unicode_encode(
-""
-if groups is None
-else ",".join(str(group) for group in
groups)
-),
-_unicode_encode(
-"" if umask is None else str(umask)
-),
-_unicode_encode(
-",".join(str(fd) for fd in fd_pipes)
-),
-binary,
-]
-+ myargs,
-)
-uid = None
-gid = None
-groups = None
-umask = None
-
-# Use _start_fork for os.fork() error handling,
ensuring
-# that if exec fails then the child proces
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/emerge/
commit: 01d06eb1d9dc8c4b16cbc9a6567ed0c07df5901a
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 24 02:45:58 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 24 03:28:24 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=01d06eb1
Fix python 3.9 CI jobs since 92ff02b9189f
Use emerge -e to fix this error we've seen since
92ff02b9189f for python 3.9 CI jobs:
https://github.com/gentoo/portage/actions/runs/8014796128/job/21893963019
test_portage_baseline[binhost emerge-gpkg] - AssertionError: 'emerge' failed
with args '('-e', '--getbinpkgonly', 'dev-libs/A')'
emerge: there are no binary packages to satisfy "dev-libs/B[flag]".
(dependency required by "dev-libs/A-1::test_repo" [binary])
(dependency required by "dev-libs/A" [argument])
Fixes: 92ff02b9189f ("emerge: Skip installed packages with emptytree in
depgraph selection")
Bug: https://bugs.gentoo.org/651018
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/emerge/conftest.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/tests/emerge/conftest.py
b/lib/portage/tests/emerge/conftest.py
index d9aec7041e..580d1e09ab 100644
--- a/lib/portage/tests/emerge/conftest.py
+++ b/lib/portage/tests/emerge/conftest.py
@@ -805,7 +805,7 @@ def _generate_all_baseline_commands(playground, binhost):
test_commands["binhost emerge"] = Noop()
else:
# The next emerge has been added to split this test from the rest:
-make_package = Emerge("--buildpkg", "dev-libs/A")
+make_package = Emerge("-e", "--buildpkg", "dev-libs/A")
getbinpkgonly = Emerge(
"-e",
"--getbinpkgonly",
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/, lib/portage/util/_eventloop/
commit: 3cc986f87ddda86ee93770e03cca06346aee54c5
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 23 06:06:14 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 23 06:48:29 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3cc986f8
AsyncioEventLoop: Call process.run_exitfuncs() before close
For the event loop running in the main thread, call
process.run_exitfuncs() before close with the event loop
running so that anything attached can clean itself up (like
the socks5 ProxyManager for bug 925240). This is necessary
because process.spawn uses the event loop to implement the
new returnproc parameter related to bug 916566.
Bug: https://bugs.gentoo.org/916566
Bug: https://bugs.gentoo.org/925240
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_socks5.py | 51 ++-
lib/portage/util/_eventloop/asyncio_event_loop.py | 44 +++
lib/portage/util/socks5.py| 16 ++-
3 files changed, 101 insertions(+), 10 deletions(-)
diff --git a/lib/portage/tests/util/test_socks5.py
b/lib/portage/tests/util/test_socks5.py
index 7b33cb3f6b..4a6d08169d 100644
--- a/lib/portage/tests/util/test_socks5.py
+++ b/lib/portage/tests/util/test_socks5.py
@@ -12,6 +12,8 @@ import time
import portage
from portage.tests import TestCase
from portage.util import socks5
+from portage.util.futures.executor.fork import ForkExecutor
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.const import PORTAGE_BIN_PATH
from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -189,8 +191,10 @@ class Socks5ServerTestCase(TestCase):
path = "/index.html"
proxy = None
tempdir = tempfile.mkdtemp()
+previous_exithandlers = portage.process._exithandlers
try:
+portage.process._exithandlers = []
with AsyncHTTPServer(host, {path: content}, loop) as server:
settings = {
"PORTAGE_TMPDIR": tempdir,
@@ -211,5 +215,50 @@ class Socks5ServerTestCase(TestCase):
self.assertEqual(result, content)
finally:
-await socks5.proxy.stop()
+try:
+# Also run_exitfuncs to test atexit hook cleanup.
+await socks5.proxy.stop()
+self.assertNotEqual(portage.process._exithandlers, [])
+portage.process.run_exitfuncs()
+self.assertEqual(portage.process._exithandlers, [])
+finally:
+portage.process._exithandlers = previous_exithandlers
+shutil.rmtree(tempdir)
+
+
+class Socks5ServerLoopCloseTestCase(TestCase):
+"""
+For bug 925240, test that the socks5 proxy is automatically
+terminated when the main event loop is closed, using a subprocess
+for isolation.
+"""
+
+def testSocks5ServerLoopClose(self):
+asyncio.run(self._testSocks5ServerLoopClose())
+
+async def _testSocks5ServerLoopClose(self):
+loop = asyncio.get_running_loop()
+self.assertEqual(
+await loop.run_in_executor(
+ForkExecutor(loop=loop),
self._testSocks5ServerLoopCloseSubprocess
+),
+True,
+)
+
+@staticmethod
+def _testSocks5ServerLoopCloseSubprocess():
+loop = global_event_loop()
+tempdir = tempfile.mkdtemp()
+try:
+settings = {
+"PORTAGE_TMPDIR": tempdir,
+"PORTAGE_BIN_PATH": PORTAGE_BIN_PATH,
+}
+
+socks5.get_socks5_proxy(settings)
+loop.run_until_complete(socks5.proxy.ready())
+finally:
+loop.close()
shutil.rmtree(tempdir)
+
+return not socks5.proxy.is_running()
diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py
b/lib/portage/util/_eventloop/asyncio_event_loop.py
index b9e96bb20e..ee9e4c60ef 100644
--- a/lib/portage/util/_eventloop/asyncio_event_loop.py
+++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
@@ -1,8 +1,9 @@
-# Copyright 2018-2023 Gentoo Authors
+# Copyright 2018-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
import signal
+import threading
import asyncio as _real_asyncio
from asyncio.events import AbstractEventLoop as _AbstractEventLoop
@@ -14,6 +15,7 @@ except ImportError:
PidfdChildWatcher = None
import portage
+from portage.util import socks5
class AsyncioEventLoop(_AbstractEventLoop):
@@ -25,18 +27,14 @@ class AsyncioEventLoop(_AbstractEventLoop):
def __init__(self, loop=None):
loop = loop or _real_asyncio.get_event_loop()
self._loop = loop
-self.run_until_complete = (
-self._run_until_complete
-if portage._internal_caller
-else loop.run_until_complete
-)
+self.run_until_complete = self._run_until_complete
[gentoo-commits] proj/portage:master commit in: /, lib/portage/tests/resolver/, lib/_emerge/
commit: 92ff02b9189f8350f44e134d538319e4037f3f71
Author: Gábor Oszkár Dénes protonmail com>
AuthorDate: Wed Feb 14 17:40:24 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Feb 23 04:39:26 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=92ff02b9
emerge: Skip installed packages with emptytree in depgraph selection
Running emerge with emptytree tries to find the best match for every
atom it needs to install. Sometimes the best matches would be
already installed packages (with `operation=nomerge`), but these
packages would be silently skipped with full emptytree installation.
This change makes sure that emerge attempts to install every package.
If the package has unmet requirements, emerge will complain.
Bug: https://bugs.gentoo.org/651018
Signed-off-by: Gábor Oszkár Dénes protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1272
Signed-off-by: Sam James gentoo.org>
NEWS | 6 +
lib/_emerge/depgraph.py| 13 ++
lib/portage/tests/resolver/test_depth.py | 8 +-
.../test_emptytree_reinstall_unsatisfiability.py | 137 +
lib/portage/tests/resolver/test_useflags.py| 6 +-
5 files changed, 166 insertions(+), 4 deletions(-)
diff --git a/NEWS b/NEWS
index 3fbc727861..94be26de84 100644
--- a/NEWS
+++ b/NEWS
@@ -6,6 +6,12 @@ Release notes take the form of the following optional
categories:
* Bug fixes
* Cleanups
+portage-3.0.63 (UNRELEASED)
+--
+
+Bug fixes:
+* emerge: Skip installed packages with emptytree in depgraph selection (bug
#651018).
+
portage-3.0.62 (2024-02-22)
--
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 70b83ee1f4..ea96bd58c4 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -7639,6 +7639,19 @@ class depgraph:
if pkg.installed and root_slot in
self._rebuild.reinstall_list:
continue
+if (
+empty
+and pkg.installed
+and not
self._frozen_config.excluded_pkgs.findAtomForPackage(
+pkg, modified_use=self._pkg_use_enabled(pkg)
+)
+):
+# With --emptytree option we assume no packages
+# are installed, so we do not select them.
+# But we allow installed packages to satisfy
dependency requirements
+# if they're explicitly excluded, so we allow them to
be selected.
+continue
+
if (
not pkg.installed
and
self._frozen_config.excluded_pkgs.findAtomForPackage(
diff --git a/lib/portage/tests/resolver/test_depth.py
b/lib/portage/tests/resolver/test_depth.py
index 9c5289f7d0..ab5f8e7ec3 100644
--- a/lib/portage/tests/resolver/test_depth.py
+++ b/lib/portage/tests/resolver/test_depth.py
@@ -1,4 +1,4 @@
-# Copyright 2011-2020 Gentoo Authors
+# Copyright 2011-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -318,6 +318,12 @@ class ResolverDepthTestCase(TestCase):
"sys-fs/udev-164",
],
),
+ResolverPlaygroundTestCase(
+["@world"],
+options={"--emptytree": True, "--exclude": ["dev-libs/B"]},
+success=True,
+mergelist=["dev-libs/C-2", "dev-libs/A-2"],
+),
)
playground = ResolverPlayground(
diff --git
a/lib/portage/tests/resolver/test_emptytree_reinstall_unsatisfiability.py
b/lib/portage/tests/resolver/test_emptytree_reinstall_unsatisfiability.py
new file mode 100644
index 00..fcdc01d7f1
--- /dev/null
+++ b/lib/portage/tests/resolver/test_emptytree_reinstall_unsatisfiability.py
@@ -0,0 +1,137 @@
+# Copyright 2024 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+ResolverPlayground,
+ResolverPlaygroundTestCase,
+)
+
+
+class EmptytreeReinstallUnsatisfiabilityTestCase(TestCase):
+def testEmptytreeReinstallUnsatisfiability(self):
+"""
+Tests to check if emerge fails and complains when --emptytree
+package dependency graph reinstall is unsatisfied, even if the already
+installed packages successfully satisfy the dependency tree.
+
+See bug #651018 where emerge silently skips package
+reinstalls because of unsatisfied use flag requirements.
+"""
+ebuilds = {
+"dev-libs/A-1": {
+"DEPEND": "dev-libs/B",
+"RDEPEND": "dev-libs/B",
+"EAPI": "2",
+},
+
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: d718cea94a180042b2285698b2c19113c5d25987 Author: Zac Medico gentoo org> AuthorDate: Thu Feb 22 06:41:49 2024 + Commit: Zac Medico gentoo org> CommitDate: Thu Feb 22 07:28:38 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d718cea9 _get_running_loop: Support real asyncio.run When called via the real asyncio.run implementation, wrap the running asyncio loop. Otherwise, it's not possible to call portage libraries via the real asyncio.run without triggering Future "attached to a different loop" errors. Bug: https://bugs.gentoo.org/761538 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/_asyncio/__init__.py | 28 +-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index 22241f335d..4eecc46a89 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -325,13 +325,37 @@ def _safe_loop(): def _get_running_loop(): +""" +This calls the real asyncio get_running_loop() and wraps that with +portage's internal AsyncioEventLoop wrapper. If there is no running +asyncio event loop but portage has a reference to another running +loop in this thread, then use that instead. + +This behavior enables portage internals to use the real asyncio.run +while remaining compatible with internal code that does not use the +real asyncio.run. +""" +try: +_loop = _real_asyncio.get_running_loop() +except RuntimeError: +_loop = None + with _thread_weakrefs.lock: if _thread_weakrefs.pid == portage.getpid(): try: loop = _thread_weakrefs.loops[threading.get_ident()] except KeyError: -return None -return loop if loop.is_running() else None +pass +else: +if _loop is loop._loop: +return loop +elif _loop is None: +return loop if loop.is_running() else None + +# If _loop it not None here it means it was probably a temporary +# loop created by asyncio.run, so we don't try to cache it, and +# just return a temporary wrapper. +return None if _loop is None else _AsyncioEventLoop(loop=_loop) def _thread_weakrefs_atexit():
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
commit: fbaaa4a733aaadc2744b656527756ac4e2b7ab58
Author: Zac Medico gentoo org>
AuthorDate: Thu Feb 22 06:47:33 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Feb 22 07:28:38 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fbaaa4a7
socks5: Use real asyncio.run
Use real asyncio.run to demonstrate that it is compatible with
portage internals. Since the socks5 ProxyManager uses the
process.spawn function, the internal _running_loop function
needs to return the correct loop for use in the wait method of
MultiprocessingProcess, or else it will lead to Future
"attached to a different loop" errors.
Bug: https://bugs.gentoo.org/761538
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_socks5.py | 45 +++
lib/portage/util/socks5.py| 30 ---
2 files changed, 46 insertions(+), 29 deletions(-)
diff --git a/lib/portage/tests/util/test_socks5.py
b/lib/portage/tests/util/test_socks5.py
index 987b41af29..7b33cb3f6b 100644
--- a/lib/portage/tests/util/test_socks5.py
+++ b/lib/portage/tests/util/test_socks5.py
@@ -1,6 +1,7 @@
-# Copyright 2019-2021 Gentoo Authors
+# Copyright 2019-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import asyncio
import functools
import shutil
import socket
@@ -10,7 +11,6 @@ import time
import portage
from portage.tests import TestCase
-from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util import socks5
from portage.const import PORTAGE_BIN_PATH
@@ -88,18 +88,20 @@ class AsyncHTTPServerTestCase(TestCase):
if f is not None:
f.close()
-def test_http_server(self):
+async def _test_http_server(self):
+asyncio.run(self._test_http_server())
+
+async def _test_http_server(self):
host = "127.0.0.1"
content = b"Hello World!\n"
path = "/index.html"
-loop = global_event_loop()
+
+loop = asyncio.get_running_loop()
for i in range(2):
with AsyncHTTPServer(host, {path: content}, loop) as server:
for j in range(2):
-result = loop.run_until_complete(
-loop.run_in_executor(
-None, self._fetch_directly, host,
server.server_port, path
-)
+result = await loop.run_in_executor(
+None, self._fetch_directly, host, server.server_port,
path
)
self.assertEqual(result, content)
@@ -177,7 +179,10 @@ class Socks5ServerTestCase(TestCase):
return f.read()
def test_socks5_proxy(self):
-loop = global_event_loop()
+asyncio.run(self._test_socks5_proxy())
+
+async def _test_socks5_proxy(self):
+loop = asyncio.get_running_loop()
host = "127.0.0.1"
content = b"Hello World!"
@@ -193,20 +198,18 @@ class Socks5ServerTestCase(TestCase):
}
proxy = socks5.get_socks5_proxy(settings)
-loop.run_until_complete(socks5.proxy.ready())
-
-result = loop.run_until_complete(
-loop.run_in_executor(
-None,
-self._fetch_via_proxy,
-proxy,
-host,
-server.server_port,
-path,
-)
+await socks5.proxy.ready()
+
+result = await loop.run_in_executor(
+None,
+self._fetch_via_proxy,
+proxy,
+host,
+server.server_port,
+path,
)
self.assertEqual(result, content)
finally:
-socks5.proxy.stop()
+await socks5.proxy.stop()
shutil.rmtree(tempdir)
diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py
index 6c68ff4106..74592aeefe 100644
--- a/lib/portage/util/socks5.py
+++ b/lib/portage/util/socks5.py
@@ -2,15 +2,16 @@
# Copyright 2015-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import asyncio
import errno
import os
import socket
+from typing import Union
import portage.data
from portage import _python_interpreter
from portage.data import portage_gid, portage_uid, userpriv_groups
from portage.process import atexit_register, spawn
-from portage.util.futures import asyncio
class ProxyManager:
@@ -57,23 +58,36 @@ class ProxyManager:
**spawn_kwargs,
)
-def stop(self):
+def stop(self) -> Union[None, asyncio.Future]:
"""
Stop the SOCKSv5 server.
+
+If there is a running asyncio event loop then asyncio.Future is
+returned which should be used to wait fo
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/package/ebuild/, ...
commit: 18cdb6331a66c1cc92f296b1aaf0538f63586875
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 08:44:50 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=18cdb633
EbuildPhase: async_check_locale
Change config.environ() check_locale calls to async_check_locale
calls in the EbuildPhase _async_start method in order to eliminate
synchronous waiting for child processes in the main event loop
thread.
Bug: https://bugs.gentoo.org/923841
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py| 4
lib/_emerge/EbuildPhase.py| 28 ++-
lib/portage/package/ebuild/config.py | 26 +++--
lib/portage/util/futures/_asyncio/__init__.py | 9 +
lib/portage/util/locale.py| 28 ++-
5 files changed, 70 insertions(+), 25 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index fc64d84c94..54177840c7 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,6 +8,7 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"_emerge.EbuildPhase:_setup_locale",
"portage.package.ebuild._metadata_invalid:eapi_invalid",
)
from portage import os
@@ -83,6 +84,9 @@ class EbuildMetadataPhase(SubProcess):
settings.setcpv(self.cpv)
settings.configdict["pkg"]["EAPI"] = parsed_eapi
+# This requires above setcpv and EAPI setup.
+await _setup_locale(self.settings)
+
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 3b366f39c7..b472803438 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import (
_prepare_fake_distdir,
_prepare_fake_filesdir,
)
+from portage.eapi import _get_eapi_attrs
from portage.util import writemsg, ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport(
+ "_post_src_install_write_metadata,"
+ "_preinst_bsdflags",
"portage.util.futures.unix_events:_set_nonblocking",
+"portage.util.locale:async_check_locale,split_LC_ALL",
)
from portage import os
from portage import _encodings
from portage import _unicode_encode
+async def _setup_locale(settings):
+eapi_attrs = _get_eapi_attrs(settings["EAPI"])
+if eapi_attrs.posixish_locale:
+split_LC_ALL(settings)
+settings["LC_COLLATE"] = "C"
+# check_locale() returns None when check can not be executed.
+if await async_check_locale(silent=True, env=settings.environ()) is
False:
+# try another locale
+for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
+settings["LC_CTYPE"] = l
+if await async_check_locale(silent=True,
env=settings.environ()):
+# TODO: output the following only once
+# writemsg(
+# _("!!! LC_CTYPE unsupported, using %s instead\n")
+# % self.settings["LC_CTYPE"]
+# )
+break
+else:
+raise AssertionError("C locale did not pass the test!")
+
+
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") +
("_ebuild_lock",)
@@ -95,6 +118,9 @@ class EbuildPhase(CompositeTask):
self._start_task(AsyncTaskFuture(future=future),
self._async_start_exit)
async def _async_start(self):
+
+await _setup_locale(self.settings)
+
need_builddir = self.phase not in
EbuildProcess._phases_without_builddir
if need_builddir:
diff --git a/lib/portage/package/ebuild/config.py
b/lib/portage/package/ebuild/config.py
index c89354cbf7..bafdc55a08 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -29,7 +29,6 @@ portage.proxy.lazyimport.lazyimport(
"portage.dbapi.vartree:vartree",
"portage.package.ebuild.doebuild:_phase_func_map",
"portage.util.compression_probe:_compressors",
-"portage.util.locale:check_locale,split_LC_ALL",
)
from portage import bsd_chflags, load_mod, os, selinux, _unicode_decode
from portage.const import (
@@ -3371,20 +3370,17 @@ class config:
mydict["EBUILD_PHASE_FUNC"] = phase_func
if eapi_attrs.posixish_locale:
-split_LC_ALL(mydict)
-
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: a42c2164ada634262ae1f791ad60298fe3468a94
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 03:39:35 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a42c2164
asyncio: Wrap asyncio.Lock for python 3.9 compat
Wrap asyncio.Lock for compatibility with python 3.9 where the
deprecated loop parameter is required in order to avoid "got
Future attached to a different loop" errors.
The pordbapi async_aux_get method can use asyncio.Lock to
serialize access to its doebuild_settings attribute in order
to prevent issues like bug 924319.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/_asyncio/__init__.py | 17 +
1 file changed, 17 insertions(+)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index 8f1b8e8275..b6481c281e 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -9,6 +9,7 @@ __all__ = (
"CancelledError",
"Future",
"InvalidStateError",
+"Lock",
"TimeoutError",
"get_child_watcher",
"get_event_loop",
@@ -22,6 +23,7 @@ __all__ = (
"wait_for",
)
+import sys
import types
import weakref
@@ -35,6 +37,7 @@ from asyncio import (
FIRST_EXCEPTION,
Future,
InvalidStateError,
+Lock as _Lock,
shield,
TimeoutError,
wait_for,
@@ -159,6 +162,20 @@ def iscoroutinefunction(func):
return False
+class Lock(_Lock):
+"""
+Inject loop parameter for python3.9 or less in order to avoid
+"got Future attached to a different loop" errors.
+"""
+
+def __init__(self, **kwargs):
+if sys.version_info >= (3, 10):
+kwargs.pop("loop", None)
+elif "loop" not in kwargs:
+kwargs["loop"] = _safe_loop()._loop
+super().__init__(**kwargs)
+
+
class Task(Future):
"""
Schedule the execution of a coroutine: wrap it in a future. A task
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/, lib/portage/tests/update/, lib/portage/dbapi/
commit: 389bb304abf5705b9f0e9e75982a682f193af985
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 04:35:02 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=389bb304
async_aux_get: Use EbuildMetadataPhase deallocate_config future
For the portdbapi async_aux_get method, there is not a very
good place to store a config pool, so instead use asyncio.Lock
to manage access to the portdbapi doebuild_settings attribute
when using the main event loop in the main thread. For other
threads, clone a config instance since we do not have a
thread-safe config pool. This cloning is expensive, but since
portage internals do not trigger this case, it suffices for now
(an AssertionError ensures that internals do not trigger it).
For the main event loop running in the main thread, performance
with the asyncio.Lock should not be significantly different to
performance prior to commit c95fc64abf96, since check_locale
results are typically cached and before there was only a single
shared doebuild_settings instance with access serialized via
the EbuildMetadataPhase _start method.
Update async_aux_get callers to use asyncio.ensure_future on
the returned coroutine when needed, since it used to return
a future instead of a coroutine, and sometimes a future is
needed for add_done_callback usage.
In the portdbapi async_fetch_map method, fix a broken reference
to "future" which should have been "aux_get_future", an error
discovered while testing this patch.
Bug: https://bugs.gentoo.org/924319
Fixes: c95fc64abf96 ("EbuildPhase: async_check_locale")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_emirrordist/FetchIterator.py | 10 ++-
lib/portage/dbapi/porttree.py | 129 --
lib/portage/tests/update/test_move_ent.py | 3 +
3 files changed, 97 insertions(+), 45 deletions(-)
diff --git a/lib/portage/_emirrordist/FetchIterator.py
b/lib/portage/_emirrordist/FetchIterator.py
index eaf3e53596..e4fdd092af 100644
--- a/lib/portage/_emirrordist/FetchIterator.py
+++ b/lib/portage/_emirrordist/FetchIterator.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2018 Gentoo Foundation
+# Copyright 2013-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import threading
@@ -14,6 +14,7 @@ from portage.exception import PortageException,
PortageKeyError
from portage.package.ebuild.fetch import DistfileName
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.TaskScheduler import TaskScheduler
+from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from .FetchTask import FetchTask
from _emerge.CompositeTask import CompositeTask
@@ -276,8 +277,11 @@ def _async_fetch_tasks(config, hash_filter, repo_config,
digests_future, cpv, lo
result.set_result(fetch_tasks)
def future_generator():
-yield config.portdb.async_aux_get(
-cpv, ("RESTRICT",), myrepo=repo_config.name, loop=loop
+yield asyncio.ensure_future(
+config.portdb.async_aux_get(
+cpv, ("RESTRICT",), myrepo=repo_config.name, loop=loop
+),
+loop,
)
yield config.portdb.async_fetch_map(cpv, mytree=repo_config.location,
loop=loop)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 61d431f917..4eebe1183f 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2021 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["close_portdbapi_caches", "FetchlistDict", "portagetree",
"portdbapi"]
@@ -41,7 +41,9 @@ from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
+import contextlib
import os as _os
+import threading
import traceback
import warnings
import errno
@@ -239,6 +241,7 @@ class portdbapi(dbapi):
# this purpose because doebuild makes many changes to the config
# instance that is passed in.
self.doebuild_settings = config(clone=self.settings)
+self._doebuild_settings_lock = asyncio.Lock()
self.depcachedir = os.path.realpath(self.settings.depcachedir)
if os.environ.get("SANDBOX_ON") == "1":
@@ -356,6 +359,17 @@ class portdbapi(dbapi):
self._better_cache = None
self._broken_ebuilds = set()
+def __getstate__(self):
+state = self.__dict__.copy()
+# These attributes are not picklable, so they are automatically
+# regenerated after unpickling.
+state["_doebuild_settings_lock"] = None
+return state
+
+def __setstate__(self, state):
+self.__dict__.update(state)
+self._doebuild_settings_lock = asyncio.L
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dbapi/, lib/portage/tests/resolver/
commit: 0dedea99ac13e0e75a83a78890ed73bced1b950b
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 04:21:26 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0dedea99
ResolverPlayground: Use egencache to create manifests
Make the ResolverPlayground _create_ebuild_manifests method
call egencache --jobs, which reliably triggers the KeyError
from bug 924319 for multiple tests:
lib/portage/tests/bin/test_doins.py::DoIns::testDoInsFallback Exception in
callback EbuildMetadataPhase._async_start_done()
handle: )>
Traceback (most recent call last):
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "lib/_emerge/EbuildMetadataPhase.py", line 154, in _async_start_done
future.cancelled() or future.result()
^^^
File "lib/_emerge/EbuildMetadataPhase.py", line 130, in _async_start
retval = portage.doebuild(
^
File "lib/portage/package/ebuild/doebuild.py", line 1030, in doebuild
doebuild_environment(
File "lib/portage/package/ebuild/doebuild.py", line 519, in
doebuild_environment
eapi = mysettings.configdict["pkg"]["EAPI"]
File "lib/portage/util/__init__.py", line 1684, in __getitem__
return UserDict.__getitem__(self, item_key)
File "lib/portage/cache/mappings.py", line 175, in __getitem__
return self.data[key]
~^
KeyError: 'EAPI'
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/dbapi/test_portdb_cache.py | 4 ++-
lib/portage/tests/resolver/ResolverPlayground.py | 38 +---
2 files changed, 23 insertions(+), 19 deletions(-)
diff --git a/lib/portage/tests/dbapi/test_portdb_cache.py
b/lib/portage/tests/dbapi/test_portdb_cache.py
index c7c6913b49..c24a4f2098 100644
--- a/lib/portage/tests/dbapi/test_portdb_cache.py
+++ b/lib/portage/tests/dbapi/test_portdb_cache.py
@@ -1,6 +1,7 @@
-# Copyright 2012-2023 Gentoo Authors
+# Copyright 2012-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import shutil
import subprocess
import sys
import textwrap
@@ -63,6 +64,7 @@ class PortdbCacheTestCase(TestCase):
python_cmd = (portage_python, "-b", "-Wd", "-c")
test_commands = (
+(lambda: shutil.rmtree(md5_cache_dir) or True,),
(lambda: not os.path.exists(pms_cache_dir),),
(lambda: not os.path.exists(md5_cache_dir),),
python_cmd
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index 2d26012873..75c86b615c 100644
--- a/lib/portage/tests/resolver/ResolverPlayground.py
+++ b/lib/portage/tests/resolver/ResolverPlayground.py
@@ -3,6 +3,7 @@
import bz2
import fnmatch
+import subprocess
import tempfile
import portage
@@ -18,8 +19,6 @@ from portage.const import (
from portage.process import find_binary
from portage.dep import Atom, _repo_separator
from portage.dbapi.bintree import binarytree
-from portage.package.ebuild.config import config
-from portage.package.ebuild.digestgen import digestgen
from portage._sets import load_default_config
from portage._sets.base import InternalPackageSet
from portage.tests import cnf_path
@@ -323,22 +322,25 @@ class ResolverPlayground:
f.write(misc_content)
def _create_ebuild_manifests(self, ebuilds):
-tmpsettings = config(clone=self.settings)
-tmpsettings["PORTAGE_QUIET"] = "1"
-for cpv in ebuilds:
-a = Atom("=" + cpv, allow_repo=True)
-repo = a.repo
-if repo is None:
-repo = "test_repo"
-
-repo_dir = self._get_repo_dir(repo)
-ebuild_dir = os.path.join(repo_dir, a.cp)
-ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] +
".ebuild")
-
-portdb = self.trees[self.eroot]["porttree"].dbapi
-tmpsettings["O"] = ebuild_dir
-if not digestgen(mysettings=tmpsettings, myportdb=portdb):
-raise AssertionError(f"digest creation failed for
{ebuild_path}")
+for repo_name in self._repositories:
+if repo_name == "DEFAULT":
+continue
+egencache_cmd = [
+"egencache",
+f"--repo={repo_name}",
+"--update",
+"--update-manifests",
+"--sign-manifests=n",
+"--strict-manifests=n",
+
f"--repositories-configuration={self.settings['PORTAGE_REPOSITORIES']}",
+f"--jobs={portage.util.cpuinfo.get_cpu_count()}",
+]
+result = subprocess.run(
+egencache_cmd,
+
[gentoo-commits] proj/portage:master commit in: lib/portage/cache/, lib/portage/tests/dbapi/
commit: 414234a218bc79564ab17312d5cc247a4c8091d7
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 03:30:00 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:30 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=414234a2
anydbm: Pickle support for multiprocessing spawn
The egencache usage in ResolverPlayground that was used to trigger
bug 924319 triggered a pickling error for AuxdbTestCase.test_anydbm
with the multiprocessing spawn start method, so fix the anydbm
cache module to omit the unpicklable database object from pickled
state, and regenerate it after unpickling.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico gentoo.org>
lib/portage/cache/anydbm.py | 17 -
lib/portage/tests/dbapi/test_auxdb.py | 4 +---
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/lib/portage/cache/anydbm.py b/lib/portage/cache/anydbm.py
index 94a270a483..ad7042ae41 100644
--- a/lib/portage/cache/anydbm.py
+++ b/lib/portage/cache/anydbm.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2020 Gentoo Authors
+# Copyright 2005-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# Author(s): Brian Harring (ferri...@gentoo.org)
@@ -67,6 +67,21 @@ class database(fs_template.FsBased):
raise cache_errors.InitializationError(self.__class__, e)
self._ensure_access(self._db_path)
+def __getstate__(self):
+state = self.__dict__.copy()
+# These attributes are not picklable, so they are automatically
+# regenerated after unpickling.
+state["_database__db"] = None
+return state
+
+def __setstate__(self, state):
+self.__dict__.update(state)
+mode = "w"
+if dbm.whichdb(self._db_path) in ("dbm.gnu", "gdbm"):
+# Allow multiple concurrent writers (see bug #53607).
+mode += "u"
+self.__db = dbm.open(self._db_path, mode, self._perms)
+
def iteritems(self):
# dbm doesn't implement items()
for k in self.__db.keys():
diff --git a/lib/portage/tests/dbapi/test_auxdb.py
b/lib/portage/tests/dbapi/test_auxdb.py
index 0de0123a5f..aac6ce361c 100644
--- a/lib/portage/tests/dbapi/test_auxdb.py
+++ b/lib/portage/tests/dbapi/test_auxdb.py
@@ -16,9 +16,7 @@ class AuxdbTestCase(TestCase):
from portage.cache.anydbm import database
except ImportError:
self.skipTest("dbm import failed")
-self._test_mod(
-"portage.cache.anydbm.database", multiproc=False, picklable=False
-)
+self._test_mod("portage.cache.anydbm.database", multiproc=False,
picklable=True)
def test_flat_hash_md5(self):
self._test_mod("portage.cache.flat_hash.md5_database")
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, bin/, man/
commit: 997058a825a340813532bef77a34425cf4a88eb2
Author: Michał Górny gentoo org>
AuthorDate: Thu Feb 15 15:33:03 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 21 02:13:45 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=997058a8
Support PROPERTIES=test_userpriv not to drop perms for tests
Support PROPERTIES=test_userpriv and a corresponding ALLOW_TEST=userpriv
to disable FEATURES=userpriv when running the test phase. This can be
used e.g. in dev-python/reflink that needs to be able to mount
filesystem on a loopback device for testing.
Bug: https://bugs.gentoo.org/924585
Signed-off-by: Michał Górny gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1274
Signed-off-by: Sam James gentoo.org>
bin/phase-functions.sh | 3 ++-
lib/portage/package/ebuild/config.py | 3 +++
lib/portage/package/ebuild/doebuild.py | 3 +++
man/ebuild.5 | 5 +
man/make.conf.5| 4
5 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index cd672a878c..ebcf5f242a 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -503,7 +503,8 @@ __dyn_test() {
fi
if has test ${PORTAGE_RESTRICT} && ! has all ${ALLOW_TEST} &&
- ! { has test_network ${PORTAGE_PROPERTIES} && has
network ${ALLOW_TEST}; }
+ ! { has test_network ${PORTAGE_PROPERTIES} && has
network ${ALLOW_TEST}; } &&
+ ! { has test_privileged ${PORTAGE_PROPERTIES} && has
privileged ${ALLOW_TEST}; }
then
einfo "Skipping make test/check due to ebuild restriction."
__vecho ">>> Test phase [disabled because of RESTRICT=test]:
${CATEGORY}/${PF}"
diff --git a/lib/portage/package/ebuild/config.py
b/lib/portage/package/ebuild/config.py
index d7b0ca5676..c89354cbf7 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -2114,6 +2114,9 @@ class config:
"test" in restrict
and not "all" in allow_test
and not ("test_network" in properties and "network" in
allow_test)
+and not (
+"test_privileged" in properties and "privileged" in
allow_test
+)
)
if restrict_test and "test" in self.features:
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 4cf155e033..bc51fdff2d 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -239,6 +239,9 @@ def _doebuild_spawn(phase, settings, actionmap=None,
**kwargs):
ebuild_sh_arg,
)
+if phase == "test" and "test_privileged" in
settings["PORTAGE_PROPERTIES"].split():
+kwargs["droppriv"] = False
+
settings["EBUILD_PHASE"] = phase
try:
return spawn(cmd, settings, **kwargs)
diff --git a/man/ebuild.5 b/man/ebuild.5
index f849f20a29..a32ba4828c 100644
--- a/man/ebuild.5
+++ b/man/ebuild.5
@@ -811,6 +811,11 @@ is installed.
The package manager may run tests that require an internet connection, even if
the ebuild has
.IR RESTRICT=test .
+.TP
+.I test_privileged
+The package manager may run tests that require superuser permissions, even if
+the ebuild has
+.IR RESTRICT=test .
.RE
.PD 1
.TP
diff --git a/man/make.conf.5 b/man/make.conf.5
index 23d8408544..e13f6eec4f 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -1323,6 +1323,10 @@ Run tests in packages specifying
\fBPROPERTIES\fR="\fBtest_network\fR". Note
that this will most likely cause Internet access during the test suite which
could cause additional costs, privacy concerns and intermittent test failures.
.TP
+.B privileged
+Run tests in packages specifying \fBPROPERTIES\fR="\fBtest_privileged\fR".
Note
+that this will cause the test suite to be run with superuser permissions.
+.TP
.RE
.TP
.B RESUMECOMMAND
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 58b4c156543705f631444040c2b962c4ac760f6a
Author: Sam James gentoo org>
AuthorDate: Wed Feb 21 02:07:59 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 21 02:08:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=58b4c156
gpkg: add missing newline to gpkg format error
Signed-off-by: Sam James gentoo.org>
lib/portage/gpkg.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index f1d8f97f8e..edb0e43fbf 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -2111,7 +2111,7 @@ class gpkg:
if self.basename and self.prefix and not
self.prefix.startswith(self.basename):
writemsg(
-colorize("WARN", f"Package basename mismatched, using
{self.prefix}")
+colorize("WARN", f"Package basename mismatched, using
{self.prefix}\n")
)
all_files = tar.getmembers()
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 14fe81d73c3330ec31bbc712ada40cc5bdda2c61
Author: Sam James gentoo org>
AuthorDate: Wed Feb 21 02:07:19 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 21 02:07:19 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=14fe81d7
binpkg: add missing newline to gpkg format error
Signed-off-by: Sam James gentoo.org>
lib/portage/binpkg.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
index 2078e3ca58..9ecd52cf3c 100644
--- a/lib/portage/binpkg.py
+++ b/lib/portage/binpkg.py
@@ -67,7 +67,7 @@ def get_binpkg_format(binpkg_path, check_file=False,
remote=False):
writemsg(
colorize(
"WARN",
-"File {} binpkg format mismatch, actual format: {}".format(
+"File {} binpkg format mismatch, actual format: {}\n".format(
binpkg_path, file_format
),
)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 038ad1029ea574b106906380e47479db1041bee2
Author: Zac Medico gentoo org>
AuthorDate: Wed Feb 14 05:55:31 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 14 06:04:22 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=038ad102
BuildLogger: Fix portage.locks._open_fds memory leak
The _file_close_wrapper __getattribute__ method needs to
be overridden to expose its close method, otherwise the
underlying file's close method is called and the closed
file object remains as a memory leak in the global
portage.locks._open_fds dict. For reference, see similar
classes like portage.util.atomic_ofstream which overrides
methods in the same way.
Bug: https://bugs.gentoo.org/919072
Fixes: df212738bbb2 ("BuildLogger: Close self._stdin after fork")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_async/BuildLogger.py | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/portage/util/_async/BuildLogger.py
b/lib/portage/util/_async/BuildLogger.py
index 9f8a21ab2b..0cfc90a942 100644
--- a/lib/portage/util/_async/BuildLogger.py
+++ b/lib/portage/util/_async/BuildLogger.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Gentoo Authors
+# Copyright 2020-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -31,6 +31,11 @@ class _file_close_wrapper(ObjectProxy):
def _get_target(self):
return object.__getattribute__(self, "_file")
+def __getattribute__(self, attr):
+if attr == "close":
+return object.__getattribute__(self, attr)
+return getattr(object.__getattribute__(self, "_file"), attr)
+
def close(self):
file = object.__getattribute__(self, "_file")
if not file.closed:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/tests/sets/files/, ...
commit: 3110ec376cbcb1f5b7fb82ba30ec958798bb32cf
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 00:46:52 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 13 05:04:40 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3110ec37
actions: Fix interaction between start-method and pytest-xdist
Use portage.test.TestCase setUp method to setup the multiprocessing
start method if needed. It needs to be done relatively late in order
to work with the pytest-xdist plugin due to execnet usage.
Bug: https://bugs.gentoo.org/914876
Signed-off-by: Zac Medico gentoo.org>
.github/workflows/ci.yml| 2 +-
lib/portage/tests/__init__.py | 12 +++-
lib/portage/tests/env/config/test_PortageModulesFile.py | 3 ++-
lib/portage/tests/news/test_NewsItem.py | 3 ++-
lib/portage/tests/sets/files/test_config_file_set.py| 3 ++-
lib/portage/tests/sets/files/test_static_file_set.py| 3 ++-
lib/portage/tests/sets/shell/test_shell.py | 4 ++--
lib/portage/tests/util/futures/test_retry.py| 1 +
8 files changed, 23 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 15da507238..5bffd97206 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -82,7 +82,6 @@ jobs:
mv "${bin_file}"{.new,}
fi
done < <(find bin -maxdepth 1 -type f)
- sed -i meson.build -e "s|'-m', 'pytest'|'-c', 'import
multiprocessing, sys, pytest; multiprocessing.set_start_method(\"spawn\",
force=True); sys.exit(pytest.console_main())'|"
- name: Test meson install --destdir /tmp/install-root
run: |
echo -e "[binaries]\npython = '$(command -v python)'" >
/tmp/native.ini
@@ -90,5 +89,6 @@ jobs:
meson install -C /tmp/build --destdir /tmp/install-root
- name: Run tests for ${{ matrix.python-version }}
run: |
+ [[ "${{ matrix.start-method }}" == "spawn" ]] && export
PORTAGE_MULTIPROCESSING_START_METHOD=spawn
export PYTEST_ADDOPTS="-vv -ra -l -o console_output_style=count -n
$(nproc) --dist=worksteal"
meson test -C /tmp/build --verbose
diff --git a/lib/portage/tests/__init__.py b/lib/portage/tests/__init__.py
index ef59852989..23dd366d89 100644
--- a/lib/portage/tests/__init__.py
+++ b/lib/portage/tests/__init__.py
@@ -1,8 +1,9 @@
# tests/__init__.py -- Portage Unit Test functionality
-# Copyright 2006-2023 Gentoo Authors
+# Copyright 2006-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import argparse
+import multiprocessing
import sys
import time
import unittest
@@ -79,6 +80,15 @@ class TestCase(unittest.TestCase):
self.bindir = cnf_bindir
self.sbindir = cnf_sbindir
+def setUp(self):
+"""
+Setup multiprocessing start method if needed. It needs to be
+done relatively late in order to work with the pytest-xdist
+plugin due to execnet usage.
+"""
+if os.environ.get("PORTAGE_MULTIPROCESSING_START_METHOD") == "spawn":
+multiprocessing.set_start_method("spawn", force=True)
+
def assertRaisesMsg(self, msg, excClass, callableObj, *args, **kwargs):
"""Fail unless an exception of class excClass is thrown
by callableObj when invoked with arguments args and keyword
diff --git a/lib/portage/tests/env/config/test_PortageModulesFile.py
b/lib/portage/tests/env/config/test_PortageModulesFile.py
index f9879df687..bca86e0e6c 100644
--- a/lib/portage/tests/env/config/test_PortageModulesFile.py
+++ b/lib/portage/tests/env/config/test_PortageModulesFile.py
@@ -1,4 +1,4 @@
-# Copyright 2006-2009 Gentoo Foundation
+# Copyright 2006-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage import os
@@ -13,6 +13,7 @@ class PortageModulesFileTestCase(TestCase):
modules = ["spanky", "zmedico", "antarus", "ricer", "5", "6"]
def setUp(self):
+super().setUp()
self.items = {}
for k, v in zip(self.keys + self.invalid_keys, self.modules):
self.items[k] = v
diff --git a/lib/portage/tests/news/test_NewsItem.py
b/lib/portage/tests/news/test_NewsItem.py
index a7903f07e4..7a8393c51f 100644
--- a/lib/portage/tests/news/test_NewsItem.py
+++ b/lib/portage/tests/news/test_NewsItem.py
@@ -1,5 +1,5 @@
# test_NewsItem.py -- Portage Unit Testing Functionality
-# Copyright 2007-2023 Gentoo Authors
+# Copyright 2007-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -114,6 +114,7 @@ class NewsItemTestCase(TestCase):
}
def setUp(self) -> None:
+super().setUp()
self.profile_base = "/var/db/repos/gentoo/profiles/default-linux"
self.profile = f"{self.profile_base}/x86/2007.0/"
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, lib/_emerge/, lib/portage/util/futures/_asyncio/, ...
commit: 5c528b1cf44f30d80a3ca5620a810e4fe2bd66f1
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 04:47:53 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 13 05:02:14 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5c528b1c
Revert "EbuildPhase: async_check_locale"
This reverts commit c95fc64abf9698263090b3ffd4a056e989dd2be1
since we had assumed EbuildMetadataPhase._start would serialize
access to the portdbapi doebuild_settings attribute, and that
assumption broke when _async_start was introduced in order to
call async_check_locale.
Bug: https://bugs.gentoo.org/923841
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py| 21
lib/_emerge/EbuildPhase.py| 28 +--
lib/portage/package/ebuild/config.py | 26 ++---
lib/portage/util/futures/_asyncio/__init__.py | 9 -
lib/portage/util/locale.py| 28 +--
5 files changed, 25 insertions(+), 87 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index 53b7ad9624..f4f685e81c 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,14 +8,12 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
-"_emerge.EbuildPhase:_setup_locale",
"portage.package.ebuild._metadata_invalid:eapi_invalid",
)
from portage import os
from portage import _encodings
from portage import _unicode_decode
from portage import _unicode_encode
-from portage.util.futures import asyncio
import fcntl
@@ -46,12 +44,6 @@ class EbuildMetadataPhase(SubProcess):
_files_dict = slot_dict_class(_file_names, prefix="")
def _start(self):
-asyncio.ensure_future(
-self._async_start(), loop=self.scheduler
-).add_done_callback(self._async_start_done)
-
-async def _async_start(self):
-
ebuild_path = self.ebuild_hash.location
with open(
@@ -83,9 +75,6 @@ class EbuildMetadataPhase(SubProcess):
settings.setcpv(self.cpv)
settings.configdict["pkg"]["EAPI"] = parsed_eapi
-# This requires above setcpv and EAPI setup.
-await _setup_locale(self.settings)
-
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
@@ -150,16 +139,6 @@ class EbuildMetadataPhase(SubProcess):
self._proc = retval
-def _async_start_done(self, future):
-future.cancelled() or future.result()
-if future.cancelled():
-self.cancel()
-self._was_cancelled()
-
-if self.returncode is not None:
-self._unregister()
-self.wait()
-
def _output_handler(self):
while True:
buf = self._read_buf(self._files.ebuild)
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index c8caf73722..c81bf54a81 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2024 Gentoo Authors
+# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -24,7 +24,6 @@ from portage.package.ebuild.prepare_build_dirs import (
_prepare_fake_distdir,
_prepare_fake_filesdir,
)
-from portage.eapi import _get_eapi_attrs
from portage.util import writemsg, ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -55,34 +54,12 @@ portage.proxy.lazyimport.lazyimport(
+ "_post_src_install_write_metadata,"
+ "_preinst_bsdflags",
"portage.util.futures.unix_events:_set_nonblocking",
-"portage.util.locale:async_check_locale,split_LC_ALL",
)
from portage import os
from portage import _encodings
from portage import _unicode_encode
-async def _setup_locale(settings):
-eapi_attrs = _get_eapi_attrs(settings["EAPI"])
-if eapi_attrs.posixish_locale:
-split_LC_ALL(settings)
-settings["LC_COLLATE"] = "C"
-# check_locale() returns None when check can not be executed.
-if await async_check_locale(silent=True, env=settings.environ()) is
False:
-# try another locale
-for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
-settings["LC_CTYPE"] = l
-if await async_check_locale(silent=True,
env=settings.environ()):
-# TODO: output the following only once
-# writemsg(
-# _("!!! LC_CTYPE unsupported, using %s instead\n")
-# % self.settings["LC_CTYPE"]
-# )
-break
-else:
-raise AssertionError("C locale did not pass the test!")
-
-
class EbuildPhase(CompositeTask):
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 419cce79f9082308c848df0a98f367de4d1c50a3
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 11 21:58:10 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 12 07:56:10 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=419cce79
process._exec: Use _start_fork for os.fork() error handling
Use _start_fork for os.fork() error handling, ensuring
that if exec fails then the child process will display
a traceback before it exits via os._exit to suppress any
finally blocks from parent's call stack (bug 345289).
Bug: https://bugs.gentoo.org/345289
Bug: https://bugs.gentoo.org/916566
Bug: https://bugs.gentoo.org/924313
Signed-off-by: Zac Medico gentoo.org>
lib/portage/process.py | 259 -
1 file changed, 151 insertions(+), 108 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 6cad250e34..f4758c824c 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -961,7 +961,13 @@ def _exec(
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
-try:
+# unshare() may not be supported by libc
+if not hasattr(libc, "unshare"):
+unshare_net = False
+unshare_ipc = False
+unshare_mount = False
+unshare_pid = False
+else:
# Since a failed unshare call could corrupt process
# state, first validate that the call can succeed.
# The parent process should call _unshare_validate
@@ -992,117 +998,154 @@ def _exec(
)
else:
if unshare_pid:
-main_child_pid = os.fork()
-if main_child_pid == 0:
-# pid namespace requires us to become init
-binary, myargs = (
-portage._python_interpreter,
-[
-portage._python_interpreter,
-os.path.join(portage._bin_path,
"pid-ns-init"),
-_unicode_encode(
-"" if uid is None else str(uid)
-),
-_unicode_encode(
-"" if gid is None else str(gid)
-),
-_unicode_encode(
-""
-if groups is None
-else ",".join(
-str(group) for group in groups
-)
-),
-_unicode_encode(
-"" if umask is None else str(umask)
-),
-_unicode_encode(
-",".join(str(fd) for fd in
fd_pipes)
-),
-binary,
-]
-+ myargs,
-)
-uid = None
-gid = None
-groups = None
-umask = None
-else:
-# Execute a supervisor process which will
forward
-# signals to init and forward exit status to
the
-# parent process. The supervisor process runs
in
-# the global pid namespace, so skip /proc
remount
-# and other setup that's intended only for the
-# init process.
-binary, myargs = portage._python_interpreter, [
+# pid namespace requires us to become init
+binary, myargs = (
+portage._python_interpreter,
+[
portage._python_interpreter,
os.path.join(portage._bin_path,
"pid-ns-init"),
-str(main_child_pid),
+_unicode_encode("" if uid is None else
str(uid)),
+_unicode_encode("" if gid is None else
str(gid)),
+
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: a1024a6d02ca3a55f86525e0d8d5089e754d3713
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 11 05:38:58 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 11 19:46:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a1024a6d
spawn_wrapper: Make pre_exec function picklable
Local functions are unpicklable, which triggered this error
with the multiprocessing "spawn" start method:
AttributeError: Can't pickle local object
'spawn_wrapper.__call__.._pre_exec'
Bug: https://bugs.gentoo.org/924273
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_selinux.py | 17 -
lib/portage/process.py | 26 ++
2 files changed, 26 insertions(+), 17 deletions(-)
diff --git a/lib/portage/_selinux.py b/lib/portage/_selinux.py
index bf6ad24895..5ae1b4e715 100644
--- a/lib/portage/_selinux.py
+++ b/lib/portage/_selinux.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# Don't use the unicode-wrapped os and shutil modules here since
@@ -6,6 +6,7 @@
import os
import shutil
import warnings
+from functools import partial
try:
import selinux
@@ -134,14 +135,12 @@ class spawn_wrapper:
def __call__(self, *args, **kwargs):
if self._con is not None:
-pre_exec = kwargs.get("pre_exec")
-
-def _pre_exec():
-if pre_exec is not None:
-pre_exec()
-setexec(self._con)
-
-kwargs["pre_exec"] = _pre_exec
+pre_exec = partial(setexec, self._con)
+kwargs["pre_exec"] = (
+portage.process._chain_pre_exec_fns(pre_exec,
kwargs["pre_exec"])
+if kwargs.get("pre_exec")
+else pre_exec
+)
return self._spawn_func(*args, **kwargs)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index a33e7b4747..6cad250e34 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -18,7 +18,7 @@ import os as _os
import warnings
from dataclasses import dataclass
-from functools import lru_cache
+from functools import lru_cache, partial
from typing import Any, Optional, Callable, Union
from portage import os
@@ -1383,18 +1383,28 @@ def _start_fork(
return pid
-class _setup_pipes_after_fork:
-def __init__(self, target, fd_pipes):
+class _chain_pre_exec_fns:
+"""
+Wraps a target function to call pre_exec functions just before
+the original target function.
+"""
+
+def __init__(self, target, *args):
self._target = target
-self._fd_pipes = fd_pipes
+self._pre_exec_fns = args
def __call__(self, *args, **kwargs):
-for fd in set(self._fd_pipes.values()):
-os.set_inheritable(fd, True)
-_setup_pipes(self._fd_pipes, close_fds=False, inheritable=True)
+for pre_exec in self._pre_exec_fns:
+pre_exec()
return self._target(*args, **kwargs)
+def _setup_pipes_after_fork(fd_pipes):
+for fd in set(fd_pipes.values()):
+os.set_inheritable(fd, True)
+_setup_pipes(fd_pipes, close_fds=False, inheritable=True)
+
+
def _start_proc(
target: Callable[..., None],
args: Optional[tuple[Any, ...]] = (),
@@ -1419,7 +1429,7 @@ def _start_proc(
# which ForkProcess does not handle because its target
# function does not necessarily exec.
if fd_pipes and multiprocessing.get_start_method() == "fork":
-target = _setup_pipes_after_fork(target, fd_pipes)
+target = _chain_pre_exec_fns(target, partial(_setup_pipes_after_fork,
fd_pipes))
fd_pipes = None
proc = ForkProcess(
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dep/, lib/portage/dep/
commit: 12f6056da88041f82a9c9dfc23ee0eab39077782
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 11 04:12:45 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 11 04:36:22 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=12f6056d
_overlap_dnf: deduplicate any-of blocks
Duplicate any-of blocks are eliminated since DNF expansion
of duplicates is nonsensical.
Bug: https://bugs.gentoo.org/891137
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dep/dep_check.py | 17 ++---
lib/portage/tests/dep/test_overlap_dnf.py | 31 +--
2 files changed, 39 insertions(+), 9 deletions(-)
diff --git a/lib/portage/dep/dep_check.py b/lib/portage/dep/dep_check.py
index 5ca0995a87..c361ee59e2 100644
--- a/lib/portage/dep/dep_check.py
+++ b/lib/portage/dep/dep_check.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["dep_check", "dep_eval", "dep_wordreduce", "dep_zapdeps"]
@@ -963,7 +963,8 @@ def _overlap_dnf(dep_struct):
order to minimize the number of packages chosen to satisfy cases like
"|| ( foo bar ) || ( bar baz )" as in bug #632026. Non-overlapping
groups are excluded from the conversion, since DNF leads to exponential
-explosion of the formula.
+explosion of the formula. Duplicate || groups are eliminated since
+DNF expansion of duplicates is nonsensical (bug #891137).
When dep_struct does not contain any overlapping groups, no DNF
conversion will be performed, and dep_struct will be returned as-is.
@@ -1021,7 +1022,17 @@ def _overlap_dnf(dep_struct):
if len(disjunctions) > 1:
overlap = True
# convert overlapping disjunctions to DNF
-result.extend(_dnf_convert(sorted(disjunctions.values(),
key=order_key)))
+dedup_set = set()
+unique_disjunctions = []
+for x in sorted(disjunctions.values(), key=order_key):
+dep_repr = portage.dep.paren_enclose(x, opconvert=True)
+if dep_repr not in dedup_set:
+dedup_set.add(dep_repr)
+unique_disjunctions.append(x)
+if len(unique_disjunctions) > 1:
+result.extend(_dnf_convert(unique_disjunctions))
+else:
+result.extend(unique_disjunctions)
else:
# pass through non-overlapping disjunctions
result.append(disjunctions.popitem()[1])
diff --git a/lib/portage/tests/dep/test_overlap_dnf.py
b/lib/portage/tests/dep/test_overlap_dnf.py
index 77352021a9..7fd1cfe7dd 100644
--- a/lib/portage/tests/dep/test_overlap_dnf.py
+++ b/lib/portage/tests/dep/test_overlap_dnf.py
@@ -1,4 +1,4 @@
-# Copyright 2017-2023 Gentoo Authors
+# Copyright 2017-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -51,22 +51,41 @@ class OverlapDNFTestCase(TestCase):
class DuplicateOverlapDNFTestCase(TestCase):
def testDuplicateOverlapDNF(self):
"""
-Demonstrate unnecessary DNF expansion for duplicate
-any-of blocks as in bug 891137.
+Demonstrate deduplication of any-of blocks, preventing unnecessary
+DNF expansion for duplicate any-of blocks as in bug 891137.
"""
test_cases = (
+("|| ( cat/A cat/B ) || ( cat/A cat/B )", [["||", "cat/A",
"cat/B"]]),
(
-"|| ( cat/A cat/B ) || ( cat/A cat/B )",
+"|| ( cat/A cat/B ) cat/E || ( cat/C cat/D ) || ( cat/A cat/B
)",
+["cat/E", ["||", "cat/A", "cat/B"], ["||", "cat/C", "cat/D"]],
+),
+(
+"|| ( cat/A cat/B ) cat/D || ( cat/B cat/C ) || ( cat/A cat/B
)",
[
+"cat/D",
[
"||",
-["cat/A", "cat/A"],
["cat/A", "cat/B"],
-["cat/B", "cat/A"],
+["cat/A", "cat/C"],
["cat/B", "cat/B"],
+["cat/B", "cat/C"],
],
],
),
+(
+"|| ( cat/A cat/B ) || ( cat/C cat/D ) || ( ( cat/B cat/E )
cat/F ) || ( cat/A cat/B )",
+[
+[
+"||",
+["cat/A", "cat/B", "cat/E"],
+["cat/A", "cat/F"],
+["cat/B", "cat/B", "cat/E"],
+["cat/B", "cat/F"],
+],
+["||", "cat/C", "cat/D"],
+],
+),
)
for dep_str, result in test_cases:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/
commit: e02feddd148f4a4a5854edd81b9aa67844d8956f
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 11 04:31:41 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 11 04:32:27 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e02feddd
test_gpkg_metadata_url_case: fix string format for pyupgrade
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
index 3773049227..e9f4111279 100644
--- a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
+++ b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
@@ -77,7 +77,7 @@ class test_gpkg_metadata_url_case(TestCase):
test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
meta_from_url = test_gpkg.get_metadata_url(
-"http://{0}:{1}/test.gpkg.tar".format(*server.server_address)
+"http://{}:{}/test.gpkg.tar".format(*server.server_address)
)
self.assertEqual(meta, meta_from_url)
@@ -148,7 +148,7 @@
IkCfAP49AOYjzuQPP0n5P0SGCINnAVEXN7QLQ4PurY/lt7cT2gEAq01stXjFhrz5
self.assertRaises(
InvalidSignature,
test_gpkg.get_metadata_url,
-
"http://{0}:{1}/test-2.gpkg.tar".format(*server.server_address),
+"http://{}:{}/test-2.gpkg.tar".format(*server.server_address),
)
finally:
if gpg is not None:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dep/
commit: 15c173dcea2401a13cfb3313918c77d7dbde133d Author: Zac Medico gentoo org> AuthorDate: Sun Feb 11 03:28:27 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Feb 11 03:29:23 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=15c173dc DuplicateOverlapDNFTestCase: Add test for bug 891137 Bug: https://bugs.gentoo.org/891137 Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/dep/test_overlap_dnf.py | 30 +- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/portage/tests/dep/test_overlap_dnf.py b/lib/portage/tests/dep/test_overlap_dnf.py index dfeded3b40..77352021a9 100644 --- a/lib/portage/tests/dep/test_overlap_dnf.py +++ b/lib/portage/tests/dep/test_overlap_dnf.py @@ -1,4 +1,4 @@ -# Copyright 2017 Gentoo Foundation +# Copyright 2017-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -46,3 +46,31 @@ class OverlapDNFTestCase(TestCase): _overlap_dnf(use_reduce(dep_str, token_class=Atom, opconvert=True)), result, ) + + +class DuplicateOverlapDNFTestCase(TestCase): +def testDuplicateOverlapDNF(self): +""" +Demonstrate unnecessary DNF expansion for duplicate +any-of blocks as in bug 891137. +""" +test_cases = ( +( +"|| ( cat/A cat/B ) || ( cat/A cat/B )", +[ +[ +"||", +["cat/A", "cat/A"], +["cat/A", "cat/B"], +["cat/B", "cat/A"], +["cat/B", "cat/B"], +], +], +), +) + +for dep_str, result in test_cases: +self.assertEqual( +_overlap_dnf(use_reduce(dep_str, token_class=Atom, opconvert=True)), +result, +)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/
commit: 9100b425a38f0f0631a4377da82f421ad87546b4 Author: Zac Medico gentoo org> AuthorDate: Sat Feb 10 21:01:17 2024 + Commit: Zac Medico gentoo org> CommitDate: Sat Feb 10 21:02:13 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9100b425 test_gpkg_metadata_url_case: fix pylint W0611: Unused import random (unused-import) Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py index 271d3e4d5f..3773049227 100644 --- a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py +++ b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py @@ -2,7 +2,6 @@ # Portage Unit Testing Functionality import io -import random import tarfile import tempfile from functools import partial
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/
commit: d279a3a5c4907f2ed8c7dc52d9c240ee33a35987
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 10 20:41:01 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 10 20:50:38 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d279a3a5
test_gpkg_metadata_url_case: optimize httpd port allocation
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 28 +++-
1 file changed, 8 insertions(+), 20 deletions(-)
diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
index 422ca7d3fa..271d3e4d5f 100644
--- a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
+++ b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
@@ -19,7 +19,7 @@ from portage.gpg import GPG
class test_gpkg_metadata_url_case(TestCase):
-def httpd(self, directory, port, httpd_future):
+def httpd(self, directory, httpd_future):
try:
import http.server
import socketserver
@@ -28,11 +28,11 @@ class test_gpkg_metadata_url_case(TestCase):
Handler = partial(http.server.SimpleHTTPRequestHandler,
directory=directory)
-with socketserver.TCPServer(("127.0.0.1", port), Handler) as httpd:
+with socketserver.TCPServer(("127.0.0.1", 0), Handler) as httpd:
httpd_future.set_result(httpd)
httpd.serve_forever()
-def start_http_server(self, directory, port):
+def start_http_server(self, directory):
try:
import threading
except ImportError:
@@ -40,7 +40,7 @@ class test_gpkg_metadata_url_case(TestCase):
httpd_future = Future()
server = threading.Thread(
-target=self.httpd, args=(directory, port, httpd_future),
daemon=True
+target=self.httpd, args=(directory, httpd_future), daemon=True
)
server.start()
return httpd_future.result()
@@ -59,13 +59,7 @@ class test_gpkg_metadata_url_case(TestCase):
server = None
try:
settings = playground.settings
-for _ in range(0, 5):
-port = random.randint(3, 6)
-try:
-server = self.start_http_server(tmpdir, port)
-except OSError:
-continue
-break
+server = self.start_http_server(tmpdir)
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
@@ -84,7 +78,7 @@ class test_gpkg_metadata_url_case(TestCase):
test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
meta_from_url = test_gpkg.get_metadata_url(
-"http://127.0.0.1:"; + str(port) + "/test.gpkg.tar"
+"http://{0}:{1}/test.gpkg.tar".format(*server.server_address)
)
self.assertEqual(meta, meta_from_url)
@@ -111,13 +105,7 @@ class test_gpkg_metadata_url_case(TestCase):
gpg = GPG(settings)
gpg.unlock()
-for _ in range(0, 5):
-port = random.randint(3, 6)
-try:
-server = self.start_http_server(tmpdir, port)
-except OSError:
-continue
-break
+server = self.start_http_server(tmpdir)
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
@@ -161,7 +149,7 @@
IkCfAP49AOYjzuQPP0n5P0SGCINnAVEXN7QLQ4PurY/lt7cT2gEAq01stXjFhrz5
self.assertRaises(
InvalidSignature,
test_gpkg.get_metadata_url,
-"http://127.0.0.1:"; + str(port) + "/test-2.gpkg.tar",
+
"http://{0}:{1}/test-2.gpkg.tar".format(*server.server_address),
)
finally:
if gpg is not None:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/
commit: d320211266b95dc8cbea295cb234a607246d955f
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 10 20:25:28 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 10 20:26:17 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d3202112
test_gpkg_metadata_url_case: shutdown http server daemon threads
Bug: https://bugs.gentoo.org/924192
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 15 ---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
index 857defe188..422ca7d3fa 100644
--- a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
+++ b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
@@ -7,6 +7,7 @@ import tarfile
import tempfile
from functools import partial
from os import urandom
+from concurrent.futures import Future
from portage.gpkg import gpkg
from portage import os
@@ -18,7 +19,7 @@ from portage.gpg import GPG
class test_gpkg_metadata_url_case(TestCase):
-def httpd(self, directory, port):
+def httpd(self, directory, port, httpd_future):
try:
import http.server
import socketserver
@@ -28,6 +29,7 @@ class test_gpkg_metadata_url_case(TestCase):
Handler = partial(http.server.SimpleHTTPRequestHandler,
directory=directory)
with socketserver.TCPServer(("127.0.0.1", port), Handler) as httpd:
+httpd_future.set_result(httpd)
httpd.serve_forever()
def start_http_server(self, directory, port):
@@ -36,11 +38,12 @@ class test_gpkg_metadata_url_case(TestCase):
except ImportError:
self.skipTest("threading module not exists")
+httpd_future = Future()
server = threading.Thread(
-target=self.httpd, args=(directory, port), daemon=True
+target=self.httpd, args=(directory, port, httpd_future),
daemon=True
)
server.start()
-return server
+return httpd_future.result()
def test_gpkg_get_metadata_url(self):
playground = ResolverPlayground(
@@ -53,6 +56,7 @@ class test_gpkg_metadata_url_case(TestCase):
}
)
tmpdir = tempfile.mkdtemp()
+server = None
try:
settings = playground.settings
for _ in range(0, 5):
@@ -85,6 +89,8 @@ class test_gpkg_metadata_url_case(TestCase):
self.assertEqual(meta, meta_from_url)
finally:
+if server is not None:
+server.shutdown()
shutil.rmtree(tmpdir)
playground.cleanup()
@@ -99,6 +105,7 @@ class test_gpkg_metadata_url_case(TestCase):
)
tmpdir = tempfile.mkdtemp()
gpg = None
+server = None
try:
settings = playground.settings
gpg = GPG(settings)
@@ -159,5 +166,7 @@
IkCfAP49AOYjzuQPP0n5P0SGCINnAVEXN7QLQ4PurY/lt7cT2gEAq01stXjFhrz5
finally:
if gpg is not None:
gpg.stop()
+if server is not None:
+server.shutdown()
shutil.rmtree(tmpdir)
playground.cleanup()
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 03be12ec5f46629fa928e5fcd45d3fe6745d5d0a
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 22:12:02 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 10 06:08:59 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=03be12ec
GPG: Use threading.Event for thread safety
Use threading.Event for thread safety during GPG stop, and use the
wait method to improve responsiveness for stop requests.
Bug: https://bugs.gentoo.org/924192
Signed-off-by: Zac Medico gentoo.org>
lib/portage/gpg.py | 14 --
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/lib/portage/gpg.py b/lib/portage/gpg.py
index 3067872244..d8a4cfcfc4 100644
--- a/lib/portage/gpg.py
+++ b/lib/portage/gpg.py
@@ -1,10 +1,9 @@
-# Copyright 2001-2020 Gentoo Authors
+# Copyright 2001-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import subprocess
import sys
import threading
-import time
from portage import os
from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
@@ -24,6 +23,7 @@ class GPG:
"""
self.settings = settings
self.thread = None
+self._terminated = None
self.GPG_signing_base_command = self.settings.get(
"BINPKG_GPG_SIGNING_BASE_COMMAND"
)
@@ -73,6 +73,7 @@ class GPG:
self.GPG_unlock_command = shlex_split(
varexpand(self.GPG_unlock_command, mydict=self.settings)
)
+self._terminated = threading.Event()
self.thread = threading.Thread(target=self.gpg_keepalive,
daemon=True)
self.thread.start()
@@ -81,16 +82,17 @@ class GPG:
Stop keepalive thread.
"""
if self.thread is not None:
-self.keepalive = False
+self._terminated.set()
def gpg_keepalive(self):
"""
Call GPG unlock command every 5 mins to avoid the passphrase expired.
"""
count = 0
-while self.keepalive:
+while not self._terminated.is_set():
if count < 5:
-time.sleep(60)
+if self._terminated.wait(60):
+break
count += 1
continue
else:
@@ -102,5 +104,5 @@ class GPG:
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
)
-if proc.wait() != os.EX_OK:
+if proc.wait() != os.EX_OK and not self._terminated.is_set():
raise GPGException("GPG keepalive failed")
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/, bin/, lib/_emerge/
commit: d7115d18dada572b6b10d6be30d0fa7fb325a2c8
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 17:19:54 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 10 06:08:58 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d7115d18
GPG: Proactively stop to avoid "GPG keepalive failed" error in pypy ci jobs
This seems to help mitigate pypy ci job hangs like those fba76a545f2
triggered.
Bug: https://bugs.gentoo.org/924192
Signed-off-by: Zac Medico gentoo.org>
bin/quickpkg | 13 -
lib/_emerge/actions.py | 10 +++---
lib/portage/tests/gpkg/test_gpkg_gpg.py | 23 ++-
lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 5 -
4 files changed, 41 insertions(+), 10 deletions(-)
diff --git a/bin/quickpkg b/bin/quickpkg
index 8443a00e64..c688c5312e 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import argparse
@@ -341,10 +341,6 @@ def quickpkg_main(options, args, eout):
portage.settings.features.remove("xattr")
portage.settings.lock()
-if portage.settings.get("BINPKG_GPG_SIGNING_KEY", None):
-gpg = GPG(portage.settings)
-gpg.unlock()
-
infos = {}
infos["successes"] = []
infos["missing"] = []
@@ -444,11 +440,18 @@ if __name__ == "__main__":
def sigwinch_handler(signum, frame):
lines, eout.term_columns = portage.output.get_term_size()
+gpg = None
+if portage.settings.get("BINPKG_GPG_SIGNING_KEY", None):
+gpg = GPG(portage.settings)
+gpg.unlock()
+
signal.signal(signal.SIGWINCH, sigwinch_handler)
try:
retval = quickpkg_main(options, args, eout)
finally:
os.umask(old_umask)
signal.signal(signal.SIGWINCH, signal.SIG_DFL)
+if gpg is not None:
+gpg.stop()
global_event_loop().close()
sys.exit(retval)
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 2710c4856c..d36a799244 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import collections
@@ -548,8 +548,10 @@ def action_build(
mergelist_shown = True
if retval != os.EX_OK:
return retval
+return os.EX_OK
-else:
+gpg = None
+try:
if not mergelist_shown:
# If we haven't already shown the merge list above, at
# least show warnings about missed updates and such.
@@ -688,8 +690,10 @@ def action_build(
ldpath_mtimes,
autoclean=1,
)
-
return retval
+finally:
+if gpg is not None:
+gpg.stop()
def action_config(settings, trees, myopts, myfiles):
diff --git a/lib/portage/tests/gpkg/test_gpkg_gpg.py
b/lib/portage/tests/gpkg/test_gpkg_gpg.py
index a2dc92150b..d7eae4a82b 100644
--- a/lib/portage/tests/gpkg/test_gpkg_gpg.py
+++ b/lib/portage/tests/gpkg/test_gpkg_gpg.py
@@ -1,4 +1,4 @@
-# Copyright Gentoo Foundation 2006-2020
+# Copyright 2022-2024 Gentoo Authors
# Portage Unit Testing Functionality
import io
@@ -26,6 +26,7 @@ class test_gpkg_gpg_case(TestCase):
}
)
tmpdir = tempfile.mkdtemp()
+gpg = None
try:
settings = playground.settings
@@ -68,6 +69,8 @@ class test_gpkg_gpg_case(TestCase):
InvalidSignature, binpkg_2.decompress, os.path.join(tmpdir,
"test")
)
finally:
+if gpg is not None:
+gpg.stop()
shutil.rmtree(tmpdir)
playground.cleanup()
@@ -81,6 +84,7 @@ class test_gpkg_gpg_case(TestCase):
}
)
tmpdir = tempfile.mkdtemp()
+gpg = None
try:
settings = playground.settings
@@ -112,6 +116,8 @@ class test_gpkg_gpg_case(TestCase):
)
finally:
+if gpg is not None:
+gpg.stop()
shutil.rmtree(tmpdir)
playground.cleanup()
@@ -133,6 +139,7 @@ class test_gpkg_gpg_case(TestCase):
}
)
tmpdir = tempfile.mkdtemp()
+gpg = None
try:
settings = playground.settings
@@ -151,6 +158,8 @@ class test_gpkg_gpg_case(TestCase):
binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir,
"test-1.gpkg.tar"))
binpkg_2.decompress(os.path.join(tmpdir, "test"))
finally:
+if gpg is not None:
+gpg.stop()
shutil.rmtree(tmpdir)
playground.cleanup()
@@ -165,6 +
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: ad61940b03be2f24c0b54c1070a4923abe18e633
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 16:22:45 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 9 23:52:11 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ad61940b
gpkg: Less aggressive subprocess.Popen kill in order to avoid BrokenPipeError
Do not kill tar_stream_reader instances if we can successfully
close them, since that can trigger BrokenPipeError during
self.proc.stdin.close() calls, and this state is best avoided
because it's unclear how the caller should handle the error.
If a BrokenPipeError does occur then simply print a traceback
and hope that the corresponding file descriptor is closed
during garbage collection. Do not try to reverse the order
of self.proc.kill() and self.proc.stdin.close() as in commit
fba76a545f2 since that triggered pypy ci job hangs for which
no reliable solution has been found so far.
Bug: https://bugs.gentoo.org/923368
Signed-off-by: Zac Medico gentoo.org>
lib/portage/gpkg.py | 19 ++-
1 file changed, 14 insertions(+), 5 deletions(-)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 031b3f87cb..f1d8f97f8e 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -1,7 +1,8 @@
-# Copyright 2001-2020 Gentoo Authors
+# Copyright 2001-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import tarfile
+import traceback
import io
import threading
import subprocess
@@ -151,7 +152,10 @@ class tar_stream_writer:
if self.proc is not None:
self.killed = True
self.proc.kill()
-self.proc.stdin.close()
+try:
+self.proc.stdin.close()
+except BrokenPipeError:
+traceback.print_exc()
self.close()
def _cmd_read_thread(self):
@@ -213,7 +217,7 @@ class tar_stream_writer:
if self.proc is not None:
self.proc.stdin.close()
if self.proc.wait() != os.EX_OK:
-if not self.error:
+if not (self.killed or self.error):
raise CompressorOperationFailed("compression failed")
if self.read_thread.is_alive():
self.read_thread.join()
@@ -349,7 +353,10 @@ class tar_stream_reader:
if self.proc is not None:
self.killed = True
self.proc.kill()
-self.proc.stdin.close()
+try:
+self.proc.stdin.close()
+except BrokenPipeError:
+traceback.print_exc()
self.close()
def read(self, bufsize=-1):
@@ -986,11 +993,13 @@ class gpkg:
try:
image_safe = tar_safe_extract(image, "image")
image_safe.extractall(decompress_dir)
+image_tar.close()
except Exception as ex:
writemsg(colorize("BAD", "!!!Extract failed."))
raise
finally:
-image_tar.kill()
+if not image_tar.closed:
+image_tar.kill()
def update_metadata(self, metadata, new_basename=None, force=False):
"""
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/gpkg/
commit: 3dac4f892479d6215c378f761505ab3d41a4b3ef Author: Zac Medico gentoo org> AuthorDate: Fri Feb 9 23:18:11 2024 + Commit: Zac Medico gentoo org> CommitDate: Fri Feb 9 23:18:26 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3dac4f89 test_gpkg_path_case: Add missing playground cleanup Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/gpkg/test_gpkg_path.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/portage/tests/gpkg/test_gpkg_path.py b/lib/portage/tests/gpkg/test_gpkg_path.py index fc57135949..19451e2e9b 100644 --- a/lib/portage/tests/gpkg/test_gpkg_path.py +++ b/lib/portage/tests/gpkg/test_gpkg_path.py @@ -1,4 +1,4 @@ -# Copyright Gentoo Foundation 2006 +# Copyright 2022-2024 Gentoo Authors # Portage Unit Testing Functionality import tempfile @@ -308,6 +308,7 @@ class test_gpkg_path_case(TestCase): self.assertEqual(r, ()) finally: shutil.rmtree(tmpdir) +playground.cleanup() def test_gpkg_long_filename(self): playground = ResolverPlayground(
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/
commit: 8dd69569b968ca0193b131c393e70855015a50dd
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 22:19:22 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 9 22:19:35 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8dd69569
ManifestTestCase: Fix tempdir removal
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_manifest.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/portage/tests/util/test_manifest.py
b/lib/portage/tests/util/test_manifest.py
index 3412d568d7..2d41b9fc97 100644
--- a/lib/portage/tests/util/test_manifest.py
+++ b/lib/portage/tests/util/test_manifest.py
@@ -11,7 +11,8 @@ from portage.tests import TestCase
class ManifestTestCase(TestCase):
def test_simple_addFile(self):
-tempdir = Path(tempfile.mkdtemp()) / "app-portage" / "diffball"
+base_tempdir = tempfile.mkdtemp()
+tempdir = Path(base_tempdir) / "app-portage" / "diffball"
manifest = Manifest(str(tempdir), required_hashes=["SHA512",
"BLAKE2B"])
(tempdir / "files").mkdir(parents=True)
@@ -30,4 +31,4 @@ class ManifestTestCase(TestCase):
manifest.getFileData("AUX", "test.patch", "SHA512"),
"e30d069dcf284cbcb2d5685f03ca362469026b469dec4f8655d0c9a2bf317f5d9f68f61855ea403f4959bc0b9c003ae824fb9d6ab2472a739950623523af9da9",
)
-shutil.rmtree(str(tempdir))
+shutil.rmtree(base_tempdir)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/resolver/
commit: be37f0761752f13a855aed66fa6e49e2f7211a0f Author: Zac Medico gentoo org> AuthorDate: Fri Feb 9 21:36:02 2024 + Commit: Zac Medico gentoo org> CommitDate: Fri Feb 9 21:36:28 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=be37f076 EAPITestCase: Disable playground debug so tempdir is cleaned up Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/resolver/test_eapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/tests/resolver/test_eapi.py b/lib/portage/tests/resolver/test_eapi.py index 5d425ccdb9..32dcb49895 100644 --- a/lib/portage/tests/resolver/test_eapi.py +++ b/lib/portage/tests/resolver/test_eapi.py @@ -1,4 +1,4 @@ -# Copyright 2010-2020 Gentoo Authors +# Copyright 2010-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -199,7 +199,7 @@ class EAPITestCase(TestCase): mergelist=["dev-libs/A-1.0", "dev-libs/B-1.0"], ) -playground = ResolverPlayground(ebuilds=ebuilds, debug=True) +playground = ResolverPlayground(ebuilds=ebuilds) try: playground.run_TestCase(test_case) self.assertEqual(test_case.test_success, True, test_case.fail_msg)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/glsa/
commit: 9cbc53ad6c483500c949c1acd70c6cbb2d7cee86 Author: Zac Medico gentoo org> AuthorDate: Fri Feb 9 21:38:22 2024 + Commit: Zac Medico gentoo org> CommitDate: Fri Feb 9 21:38:39 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9cbc53ad SecuritySetTestCase: Disable playground debug so tempdir is cleaned up Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/glsa/test_security_set.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/tests/glsa/test_security_set.py b/lib/portage/tests/glsa/test_security_set.py index a0ba1e5b45..1206d9f80f 100644 --- a/lib/portage/tests/glsa/test_security_set.py +++ b/lib/portage/tests/glsa/test_security_set.py @@ -1,4 +1,4 @@ -# Copyright 2013-2023 Gentoo Authors +# Copyright 2013-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 @@ -226,7 +226,7 @@ class SecuritySetTestCase(TestCase): # Give each GLSA a clean slate for glsa in glsas: playground = ResolverPlayground( -ebuilds=ebuilds, installed=installed, world=world, debug=True +ebuilds=ebuilds, installed=installed, world=world, debug=False ) try:
