[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/util/file_copy/, src/
commit: 23529ee81964665107400e87fc3d49c256e915c0
Author: Mike Gilbert gentoo org>
AuthorDate: Fri Mar 1 15:45:58 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 15 20:05:34 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=23529ee8
Replace linux_reflink extension module
Python 3.8 added support for copy_file_range in the os module,
so we can just call that directly.
Also, we can use the FICLONE ioctl for fast file clones on supported
filesystems (btrfs).
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/file_copy.py | 137 ++
lib/portage/util/file_copy/__init__.py | 36 ---
lib/portage/util/file_copy/meson.build | 7 -
lib/portage/util/meson.build | 2 +-
src/meson.build| 20 --
src/portage_util_file_copy_reflink_linux.c | 396 -
6 files changed, 138 insertions(+), 460 deletions(-)
diff --git a/lib/portage/util/file_copy.py b/lib/portage/util/file_copy.py
new file mode 100644
index 00..e3926d8ef6
--- /dev/null
+++ b/lib/portage/util/file_copy.py
@@ -0,0 +1,137 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import fcntl
+import logging
+import os
+import platform
+import shutil
+import sys
+
+
+logger = logging.getLogger(__name__)
+
+# Added in Python 3.12
+FICLONE = getattr(fcntl, "FICLONE", 0x40049409)
+
+# Unavailable in PyPy
+SEEK_DATA = getattr(os, "SEEK_DATA", 3)
+SEEK_HOLE = getattr(os, "SEEK_HOLE", 4)
+
+
+def _get_chunks(src):
+try:
+offset_hole = 0
+while True:
+try:
+# Find the next bit of data
+offset_data = os.lseek(src, offset_hole, SEEK_DATA)
+except OSError as e:
+# Re-raise for unexpected errno values
+if e.errno not in (errno.EINVAL, errno.ENXIO):
+raise
+
+offset_end = os.lseek(src, 0, os.SEEK_END)
+
+if e.errno == errno.ENXIO:
+# End of file
+if offset_end > offset_hole:
+# Hole at end of file
+yield (offset_end, 0)
+else:
+# SEEK_DATA failed with EINVAL, return the whole file
+yield (0, offset_end)
+
+break
+else:
+offset_hole = os.lseek(src, offset_data, SEEK_HOLE)
+yield (offset_data, offset_hole - offset_data)
+
+except OSError:
+logger.warning("_get_chunks failed unexpectedly",
exc_info=sys.exc_info())
+raise
+
+
+def _do_copy_file_range(src, dst, offset, count):
+while count > 0:
+# count must fit in ssize_t
+c = min(count, sys.maxsize)
+written = os.copy_file_range(src, dst, c, offset, offset)
+if written == 0:
+# https://bugs.gentoo.org/828844
+raise OSError(errno.EOPNOTSUPP, os.strerror(errno.EOPNOTSUPP))
+offset += written
+count -= written
+
+
+def _do_sendfile(src, dst, offset, count):
+os.lseek(dst, offset, os.SEEK_SET)
+while count > 0:
+# count must fit in ssize_t
+c = min(count, sys.maxsize)
+written = os.sendfile(dst, src, offset, c)
+offset += written
+count -= written
+
+
+def _fastcopy(src, dst):
+with (
+open(src, "rb", buffering=0) as srcf,
+open(dst, "wb", buffering=0) as dstf,
+):
+srcfd = srcf.fileno()
+dstfd = dstf.fileno()
+
+if platform.system() == "Linux":
+try:
+fcntl.ioctl(dstfd, FICLONE, srcfd)
+return
+except OSError:
+pass
+
+try_cfr = hasattr(os, "copy_file_range")
+
+for offset, count in _get_chunks(srcfd):
+if count == 0:
+os.ftruncate(dstfd, offset)
+else:
+if try_cfr:
+try:
+_do_copy_file_range(srcfd, dstfd, offset, count)
+continue
+except OSError as e:
+try_cfr = False
+if e.errno not in (errno.EXDEV, errno.ENOSYS,
errno.EOPNOTSUPP):
+logger.warning(
+"_do_copy_file_range failed unexpectedly",
+exc_info=sys.exc_info(),
+)
+try:
+_do_sendfile(srcfd, dstfd, offset, count)
+except OSError:
+logger.warning(
+"_do_sendfile failed unexpectedly",
exc_info=sys.exc_info()
+)
+raise
+
+
+def copyfile(src, dst):
+"""
+Copy the contents (no metadata) of the file named src to a file
+named dst.
+
+If possible,
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: b7e89f866a9a1d73ab72670d74e2292b05893849
Author: Mike Gilbert gentoo org>
AuthorDate: Wed Mar 6 04:01:27 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Wed Mar 6 18:19:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7e89f86
util: set a timeout for urlopen calls
A hung urlopen call can cause emerge to produce no output when fetching
binhost data.
Bug: https://bugs.gentoo.org/926221
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/_urlopen.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/portage/util/_urlopen.py b/lib/portage/util/_urlopen.py
index 22f0e08df0..d451a94a89 100644
--- a/lib/portage/util/_urlopen.py
+++ b/lib/portage/util/_urlopen.py
@@ -26,10 +26,10 @@ def have_pep_476():
return hasattr(__import__("ssl"), "_create_unverified_context")
-def urlopen(url, if_modified_since=None, headers={}, proxies=None):
+def urlopen(url, timeout=10, if_modified_since=None, headers={}, proxies=None):
parse_result = urllib_parse.urlparse(url)
if parse_result.scheme not in ("http", "https"):
-return _urlopen(url)
+return _urlopen(url, timeout=timeout)
netloc = parse_result.netloc.rpartition("@")[-1]
url = urllib_parse.urlunparse(
@@ -59,7 +59,7 @@ def urlopen(url, if_modified_since=None, headers={},
proxies=None):
handlers.append(urllib_request.ProxyHandler(proxies))
opener = urllib_request.build_opener(*handlers)
-hdl = opener.open(request)
+hdl = opener.open(request, timeout=timeout)
if hdl.headers.get("last-modified", ""):
try:
add_header = hdl.headers.add_header
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/, lib/portage/util/, lib/portage/tests/util/
commit: 11c65496bd951c3b7778a3c1ea240e347b1f4c38 Author: Zac Medico gentoo org> AuthorDate: Sun Mar 3 21:06:13 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 3 21:10:49 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=11c65496 socks5: Use run_coroutine_exitfuncs() Since commit c3ebdbb42e72 the atexit_register(proxy.stop) call in the get_socks5_proxy function can accept a coroutine function to execute in run_coroutine_exitfuncs(), so convert the ProxyManager stop method to a coroutine and use run_coroutine_exitfuncs(). Bug: https://bugs.gentoo.org/925240 Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/test_socks5.py | 5 ++- lib/portage/util/_eventloop/asyncio_event_loop.py | 12 --- lib/portage/util/socks5.py| 39 +++ 3 files changed, 7 insertions(+), 49 deletions(-) diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py index 4a6d08169d..a8cd0c46c4 100644 --- a/lib/portage/tests/util/test_socks5.py +++ b/lib/portage/tests/util/test_socks5.py @@ -216,10 +216,9 @@ class Socks5ServerTestCase(TestCase): self.assertEqual(result, content) finally: try: -# Also run_exitfuncs to test atexit hook cleanup. -await socks5.proxy.stop() +# Also run_coroutine_exitfuncs to test atexit hook cleanup. self.assertNotEqual(portage.process._exithandlers, []) -portage.process.run_exitfuncs() +await portage.process.run_coroutine_exitfuncs() self.assertEqual(portage.process._exithandlers, []) finally: portage.process._exithandlers = previous_exithandlers diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index a598b1b516..821cc7f102 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -15,7 +15,6 @@ except ImportError: PidfdChildWatcher = None import portage -from portage.util import socks5 class AsyncioEventLoop(_AbstractEventLoop): @@ -75,17 +74,6 @@ class AsyncioEventLoop(_AbstractEventLoop): self._closing = False async def _close_main(self): -# Even though this has an exit hook, invoke it here so that -# we can properly wait for it and avoid messages like this: -# [ERROR] Task was destroyed but it is pending! -if socks5.proxy.is_running(): -# TODO: Convert socks5.proxy.stop() to a regular coroutine -# function so that it doesn't need to be wrapped like this. -async def stop_socks5_proxy(): -await socks5.proxy.stop() - -portage.process.atexit_register(stop_socks5_proxy) - await portage.process.run_coroutine_exitfuncs() portage.process.run_exitfuncs() diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index f8fcdf9fca..c32ba77674 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -6,15 +6,8 @@ import asyncio import errno import os import socket -from typing import Union import portage - -portage.proxy.lazyimport.lazyimport( -globals(), -"portage.util._eventloop.global_event_loop:global_event_loop", -) - import portage.data from portage import _python_interpreter from portage.data import portage_gid, portage_uid, userpriv_groups @@ -65,41 +58,19 @@ class ProxyManager: **spawn_kwargs, ) -def stop(self) -> Union[None, asyncio.Future]: +async def stop(self): """ -Stop the SOCKSv5 server. - -If there is a running asyncio event loop then asyncio.Future is -returned which should be used to wait for the server process -to exit. +Stop the SOCKSv5 server. This method is a coroutine. """ -future = None -try: -loop = asyncio.get_running_loop() -except RuntimeError: -loop = None if self._proc is not None: self._proc.terminate() -if loop is None: -# In this case spawn internals would have used -# portage's global loop when attaching a waiter to -# self._proc, so we are obligated to use that. -global_event_loop().run_until_complete(self._proc.wait()) -else: -if self._proc_waiter is None: -self._proc_waiter = asyncio.ensure_future( -self._proc.wait(), loop=loop -) -future = asyncio.shield(self._proc_waiter) - -if loop is not None and future is None: -future = loop.create_future() -future.set_result(None) +if self._proc_waiter is None: +self._proc_waiter
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: d8089d1af39c80e1edfb1669ae92fef7ab30e08a
Author: Mike Gilbert gentoo org>
AuthorDate: Fri Mar 1 15:49:26 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 1 16:19:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8089d1a
Improve whitespace handling when parsing /proc/self/mountinfo
Only break lines on "\n" (line feed).
Only split lines on " " (space).
Bug: https://bugs.gentoo.org/925888
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/writeable_check.py | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/portage/util/writeable_check.py
b/lib/portage/util/writeable_check.py
index 3427315cf7..ad1d9edff0 100644
--- a/lib/portage/util/writeable_check.py
+++ b/lib/portage/util/writeable_check.py
@@ -47,6 +47,7 @@ def linux_ro_checker(dir_list):
"/proc/self/mountinfo",
encoding=_encodings["content"],
errors="replace",
+newline="\n",
) as f:
for line in f:
# we're interested in dir and both attr fields which always
@@ -58,7 +59,7 @@ def linux_ro_checker(dir_list):
# to the left of the ' - ', after the attr's, so split it there
mount = line.split(" - ", 1)
try:
-_dir, attr1 = mount[0].split()[4:6]
+_dir, attr1 = mount[0].split(" ")[4:6]
except ValueError:
# If it raises ValueError we can simply ignore the line.
invalids.append(line)
@@ -68,10 +69,10 @@ def linux_ro_checker(dir_list):
# for example: 16 1 0:16 / /root rw,noatime - lxfs rw
if len(mount) > 1:
try:
-attr2 = mount[1].split()[2]
+attr2 = mount[1].split(" ")[2]
except IndexError:
try:
-attr2 = mount[1].split()[1]
+attr2 = mount[1].split(" ")[1]
except IndexError:
invalids.append(line)
continue
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, src/
commit: a860484b6e8c8d107255f2105796ae7f58812e9f
Author: Mike Gilbert gentoo org>
AuthorDate: Thu Feb 29 19:38:20 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Thu Feb 29 19:38:20 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a860484b
Drop portage.util.libc extension module
This was originally added as a workaround for musl, where
ctypes.util.find_library fails.
Instead, we now try to load "libc.so" as a fallback and can just rely on
ctypes to call tolower() and toupper().
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/locale.py | 9 +++
src/meson.build| 9 ---
src/portage_util_libc.c| 67 --
3 files changed, 3 insertions(+), 82 deletions(-)
diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py
index d0edeb4afe..f45d761760 100644
--- a/lib/portage/util/locale.py
+++ b/lib/portage/util/locale.py
@@ -43,12 +43,9 @@ def _check_locale(silent):
"""
The inner locale check function.
"""
-try:
-from portage.util import libc
-except ImportError:
-(libc, _) = load_libc()
-if libc is None:
-return None
+(libc, _) = load_libc()
+if libc is None:
+return None
lc = list(range(ord("a"), ord("z") + 1))
uc = list(range(ord("A"), ord("Z") + 1))
diff --git a/src/meson.build b/src/meson.build
index cbc7aa611c..6a36724ceb 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -2,14 +2,6 @@
# and for development. Meson does not allow you to build in-place and Python
# cannot create a single namespace from two identically-named paths.
-libc_ext = py.extension_module(
-'libc',
-'portage_util_libc.c',
-dependencies : py.dependency(),
-subdir : 'portage' / 'util',
-install : true
-)
-
whirlpool_ext = py.extension_module(
'_whirlpool',
'portage_util__whirlpool.c',
@@ -21,7 +13,6 @@ whirlpool_ext = py.extension_module(
run_command(
[
'ln', '-srnf',
-libc_ext.full_path(),
whirlpool_ext.full_path(),
meson.project_source_root() / 'lib' / 'portage' / 'util/'
],
diff --git a/src/portage_util_libc.c b/src/portage_util_libc.c
deleted file mode 100644
index 12b2440c72..00
--- a/src/portage_util_libc.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/* Copyright 2005-2020 Gentoo Authors
- * Distributed under the terms of the GNU General Public License v2
- */
-
-#include
-#include
-#include
-
-static PyObject * _libc_tolower(PyObject *, PyObject *);
-static PyObject * _libc_toupper(PyObject *, PyObject *);
-
-static PyMethodDef LibcMethods[] = {
- {
- .ml_name = "tolower",
- .ml_meth = _libc_tolower,
- .ml_flags = METH_VARARGS,
- .ml_doc = "Convert to lower case using system locale."
-
- },
- {
- .ml_name = "toupper",
- .ml_meth = _libc_toupper,
- .ml_flags = METH_VARARGS,
- .ml_doc = "Convert to upper case using system locale."
- },
- {NULL, NULL, 0, NULL}
-};
-
-static struct PyModuleDef moduledef = {
- PyModuleDef_HEAD_INIT,
- .m_name = "libc",
- .m_doc = "Module for converting case using the system locale",
- .m_size = -1,
- .m_methods = LibcMethods,
-};
-
-PyMODINIT_FUNC
-PyInit_libc(void)
-{
- PyObject *m;
- m = PyModule_Create();
- return m;
-}
-
-
-static PyObject *
-_libc_tolower(PyObject *self, PyObject *args)
-{
- int c;
-
- if (!PyArg_ParseTuple(args, "i", ))
- return NULL;
-
- return Py_BuildValue("i", tolower(c));
-}
-
-
-static PyObject *
-_libc_toupper(PyObject *self, PyObject *args)
-{
- int c;
-
- if (!PyArg_ParseTuple(args, "i", ))
- return NULL;
-
- return Py_BuildValue("i", toupper(c));
-}
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/, lib/_emerge/
commit: 0ff7a3b28e0ec63d68d32e01145db8962d53774d
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 26 05:09:21 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 26 05:09:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0ff7a3b2
SpawnProcess: Wait for async set_term_size
Use the SpawnProcess _unregister method to handle async
set_term_size results, avoiding possible messages like
bug 925456 triggered:
[ERROR] Task was destroyed but it is pending!
Also update _BinpkgFetcherProcess and _EbuildFetcherProcess
which inherit the _pty_ready attribute from SpawnProcess.
Fixes: f97e414ce980 ("set_term_size: Wait asynchronously if event loop is
running")
Bug: https://bugs.gentoo.org/923750
Bug: https://bugs.gentoo.org/925456
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/BinpkgFetcher.py | 6 --
lib/_emerge/EbuildFetcher.py | 6 --
lib/_emerge/SpawnProcess.py | 14 +-
lib/portage/output.py| 13 +
lib/portage/util/_pty.py | 23 ---
5 files changed, 46 insertions(+), 16 deletions(-)
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index a1524dc009..587e4a57a3 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.AsynchronousLock import AsynchronousLock
@@ -233,7 +233,9 @@ class _BinpkgFetcherProcess(SpawnProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def sync_timestamp(self):
diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py
index 7a45d95172..81d4b1054b 100644
--- a/lib/_emerge/EbuildFetcher.py
+++ b/lib/_emerge/EbuildFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -394,7 +394,9 @@ class _EbuildFetcherProcess(ForkProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def _eerror(self, lines):
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index b63afae01c..9fc12c42e5 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -46,6 +46,7 @@ class SpawnProcess(SubProcess):
+ (
"_main_task",
"_main_task_cancel",
+"_pty_ready",
"_selinux_type",
)
)
@@ -193,6 +194,9 @@ class SpawnProcess(SubProcess):
self._main_task.add_done_callback(self._main_exit)
async def _main(self, build_logger, pipe_logger, loop=None):
+if isinstance(self._pty_ready, asyncio.Future):
+await self._pty_ready
+self._pty_ready = None
try:
if pipe_logger.poll() is None:
await pipe_logger.async_wait()
@@ -238,7 +242,9 @@ class SpawnProcess(SubProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def _spawn(
@@ -258,6 +264,12 @@ class SpawnProcess(SubProcess):
SubProcess._unregister(self)
if self._main_task is not None:
self._main_task.done() or self._main_task.cancel()
+if isinstance(self._pty_ready, asyncio.Future):
+(
+self._pty_ready.done()
+and (self._pty_ready.cancelled() or self._pty_ready.result()
or True)
+) or self._pty_ready.cancel()
+self._pty_ready = None
def _cancel(self):
if self._main_task is not None:
diff --git a/lib/portage/output.py b/lib/portage/output.py
index 7d3a6278f3..4408705c45 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -8,6 +8,7 @@ import itertools
import re
import subprocess
import sys
+from typing import Optional
import portage
@@ -554,10 +555,16 @@ def get_term_size(fd=None):
return (0, 0)
-def set_term_size(lines, columns, fd):
+def set_term_size(lines: int, columns: int, fd: int) ->
Optional[asyncio.Future]:
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: d718cea94a180042b2285698b2c19113c5d25987 Author: Zac Medico gentoo org> AuthorDate: Thu Feb 22 06:41:49 2024 + Commit: Zac Medico gentoo org> CommitDate: Thu Feb 22 07:28:38 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d718cea9 _get_running_loop: Support real asyncio.run When called via the real asyncio.run implementation, wrap the running asyncio loop. Otherwise, it's not possible to call portage libraries via the real asyncio.run without triggering Future "attached to a different loop" errors. Bug: https://bugs.gentoo.org/761538 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/_asyncio/__init__.py | 28 +-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index 22241f335d..4eecc46a89 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -325,13 +325,37 @@ def _safe_loop(): def _get_running_loop(): +""" +This calls the real asyncio get_running_loop() and wraps that with +portage's internal AsyncioEventLoop wrapper. If there is no running +asyncio event loop but portage has a reference to another running +loop in this thread, then use that instead. + +This behavior enables portage internals to use the real asyncio.run +while remaining compatible with internal code that does not use the +real asyncio.run. +""" +try: +_loop = _real_asyncio.get_running_loop() +except RuntimeError: +_loop = None + with _thread_weakrefs.lock: if _thread_weakrefs.pid == portage.getpid(): try: loop = _thread_weakrefs.loops[threading.get_ident()] except KeyError: -return None -return loop if loop.is_running() else None +pass +else: +if _loop is loop._loop: +return loop +elif _loop is None: +return loop if loop.is_running() else None + +# If _loop it not None here it means it was probably a temporary +# loop created by asyncio.run, so we don't try to cache it, and +# just return a temporary wrapper. +return None if _loop is None else _AsyncioEventLoop(loop=_loop) def _thread_weakrefs_atexit():
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
commit: fbaaa4a733aaadc2744b656527756ac4e2b7ab58
Author: Zac Medico gentoo org>
AuthorDate: Thu Feb 22 06:47:33 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Feb 22 07:28:38 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fbaaa4a7
socks5: Use real asyncio.run
Use real asyncio.run to demonstrate that it is compatible with
portage internals. Since the socks5 ProxyManager uses the
process.spawn function, the internal _running_loop function
needs to return the correct loop for use in the wait method of
MultiprocessingProcess, or else it will lead to Future
"attached to a different loop" errors.
Bug: https://bugs.gentoo.org/761538
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_socks5.py | 45 +++
lib/portage/util/socks5.py| 30 ---
2 files changed, 46 insertions(+), 29 deletions(-)
diff --git a/lib/portage/tests/util/test_socks5.py
b/lib/portage/tests/util/test_socks5.py
index 987b41af29..7b33cb3f6b 100644
--- a/lib/portage/tests/util/test_socks5.py
+++ b/lib/portage/tests/util/test_socks5.py
@@ -1,6 +1,7 @@
-# Copyright 2019-2021 Gentoo Authors
+# Copyright 2019-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import asyncio
import functools
import shutil
import socket
@@ -10,7 +11,6 @@ import time
import portage
from portage.tests import TestCase
-from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util import socks5
from portage.const import PORTAGE_BIN_PATH
@@ -88,18 +88,20 @@ class AsyncHTTPServerTestCase(TestCase):
if f is not None:
f.close()
-def test_http_server(self):
+async def _test_http_server(self):
+asyncio.run(self._test_http_server())
+
+async def _test_http_server(self):
host = "127.0.0.1"
content = b"Hello World!\n"
path = "/index.html"
-loop = global_event_loop()
+
+loop = asyncio.get_running_loop()
for i in range(2):
with AsyncHTTPServer(host, {path: content}, loop) as server:
for j in range(2):
-result = loop.run_until_complete(
-loop.run_in_executor(
-None, self._fetch_directly, host,
server.server_port, path
-)
+result = await loop.run_in_executor(
+None, self._fetch_directly, host, server.server_port,
path
)
self.assertEqual(result, content)
@@ -177,7 +179,10 @@ class Socks5ServerTestCase(TestCase):
return f.read()
def test_socks5_proxy(self):
-loop = global_event_loop()
+asyncio.run(self._test_socks5_proxy())
+
+async def _test_socks5_proxy(self):
+loop = asyncio.get_running_loop()
host = "127.0.0.1"
content = b"Hello World!"
@@ -193,20 +198,18 @@ class Socks5ServerTestCase(TestCase):
}
proxy = socks5.get_socks5_proxy(settings)
-loop.run_until_complete(socks5.proxy.ready())
-
-result = loop.run_until_complete(
-loop.run_in_executor(
-None,
-self._fetch_via_proxy,
-proxy,
-host,
-server.server_port,
-path,
-)
+await socks5.proxy.ready()
+
+result = await loop.run_in_executor(
+None,
+self._fetch_via_proxy,
+proxy,
+host,
+server.server_port,
+path,
)
self.assertEqual(result, content)
finally:
-socks5.proxy.stop()
+await socks5.proxy.stop()
shutil.rmtree(tempdir)
diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py
index 6c68ff4106..74592aeefe 100644
--- a/lib/portage/util/socks5.py
+++ b/lib/portage/util/socks5.py
@@ -2,15 +2,16 @@
# Copyright 2015-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import asyncio
import errno
import os
import socket
+from typing import Union
import portage.data
from portage import _python_interpreter
from portage.data import portage_gid, portage_uid, userpriv_groups
from portage.process import atexit_register, spawn
-from portage.util.futures import asyncio
class ProxyManager:
@@ -57,23 +58,36 @@ class ProxyManager:
**spawn_kwargs,
)
-def stop(self):
+def stop(self) -> Union[None, asyncio.Future]:
"""
Stop the SOCKSv5 server.
+
+If there is a running asyncio event loop then asyncio.Future is
+returned which should be used to wait
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/package/ebuild/, ...
commit: 18cdb6331a66c1cc92f296b1aaf0538f63586875
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 08:44:50 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=18cdb633
EbuildPhase: async_check_locale
Change config.environ() check_locale calls to async_check_locale
calls in the EbuildPhase _async_start method in order to eliminate
synchronous waiting for child processes in the main event loop
thread.
Bug: https://bugs.gentoo.org/923841
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py| 4
lib/_emerge/EbuildPhase.py| 28 ++-
lib/portage/package/ebuild/config.py | 26 +++--
lib/portage/util/futures/_asyncio/__init__.py | 9 +
lib/portage/util/locale.py| 28 ++-
5 files changed, 70 insertions(+), 25 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index fc64d84c94..54177840c7 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,6 +8,7 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"_emerge.EbuildPhase:_setup_locale",
"portage.package.ebuild._metadata_invalid:eapi_invalid",
)
from portage import os
@@ -83,6 +84,9 @@ class EbuildMetadataPhase(SubProcess):
settings.setcpv(self.cpv)
settings.configdict["pkg"]["EAPI"] = parsed_eapi
+# This requires above setcpv and EAPI setup.
+await _setup_locale(self.settings)
+
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 3b366f39c7..b472803438 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import (
_prepare_fake_distdir,
_prepare_fake_filesdir,
)
+from portage.eapi import _get_eapi_attrs
from portage.util import writemsg, ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport(
+ "_post_src_install_write_metadata,"
+ "_preinst_bsdflags",
"portage.util.futures.unix_events:_set_nonblocking",
+"portage.util.locale:async_check_locale,split_LC_ALL",
)
from portage import os
from portage import _encodings
from portage import _unicode_encode
+async def _setup_locale(settings):
+eapi_attrs = _get_eapi_attrs(settings["EAPI"])
+if eapi_attrs.posixish_locale:
+split_LC_ALL(settings)
+settings["LC_COLLATE"] = "C"
+# check_locale() returns None when check can not be executed.
+if await async_check_locale(silent=True, env=settings.environ()) is
False:
+# try another locale
+for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
+settings["LC_CTYPE"] = l
+if await async_check_locale(silent=True,
env=settings.environ()):
+# TODO: output the following only once
+# writemsg(
+# _("!!! LC_CTYPE unsupported, using %s instead\n")
+# % self.settings["LC_CTYPE"]
+# )
+break
+else:
+raise AssertionError("C locale did not pass the test!")
+
+
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") +
("_ebuild_lock",)
@@ -95,6 +118,9 @@ class EbuildPhase(CompositeTask):
self._start_task(AsyncTaskFuture(future=future),
self._async_start_exit)
async def _async_start(self):
+
+await _setup_locale(self.settings)
+
need_builddir = self.phase not in
EbuildProcess._phases_without_builddir
if need_builddir:
diff --git a/lib/portage/package/ebuild/config.py
b/lib/portage/package/ebuild/config.py
index c89354cbf7..bafdc55a08 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -29,7 +29,6 @@ portage.proxy.lazyimport.lazyimport(
"portage.dbapi.vartree:vartree",
"portage.package.ebuild.doebuild:_phase_func_map",
"portage.util.compression_probe:_compressors",
-"portage.util.locale:check_locale,split_LC_ALL",
)
from portage import bsd_chflags, load_mod, os, selinux, _unicode_decode
from portage.const import (
@@ -3371,20 +3370,17 @@ class config:
mydict["EBUILD_PHASE_FUNC"] = phase_func
if eapi_attrs.posixish_locale:
-split_LC_ALL(mydict)
-
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: a42c2164ada634262ae1f791ad60298fe3468a94
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 03:39:35 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a42c2164
asyncio: Wrap asyncio.Lock for python 3.9 compat
Wrap asyncio.Lock for compatibility with python 3.9 where the
deprecated loop parameter is required in order to avoid "got
Future attached to a different loop" errors.
The pordbapi async_aux_get method can use asyncio.Lock to
serialize access to its doebuild_settings attribute in order
to prevent issues like bug 924319.
Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/_asyncio/__init__.py | 17 +
1 file changed, 17 insertions(+)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index 8f1b8e8275..b6481c281e 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -9,6 +9,7 @@ __all__ = (
"CancelledError",
"Future",
"InvalidStateError",
+"Lock",
"TimeoutError",
"get_child_watcher",
"get_event_loop",
@@ -22,6 +23,7 @@ __all__ = (
"wait_for",
)
+import sys
import types
import weakref
@@ -35,6 +37,7 @@ from asyncio import (
FIRST_EXCEPTION,
Future,
InvalidStateError,
+Lock as _Lock,
shield,
TimeoutError,
wait_for,
@@ -159,6 +162,20 @@ def iscoroutinefunction(func):
return False
+class Lock(_Lock):
+"""
+Inject loop parameter for python3.9 or less in order to avoid
+"got Future attached to a different loop" errors.
+"""
+
+def __init__(self, **kwargs):
+if sys.version_info >= (3, 10):
+kwargs.pop("loop", None)
+elif "loop" not in kwargs:
+kwargs["loop"] = _safe_loop()._loop
+super().__init__(**kwargs)
+
+
class Task(Future):
"""
Schedule the execution of a coroutine: wrap it in a future. A task
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 038ad1029ea574b106906380e47479db1041bee2
Author: Zac Medico gentoo org>
AuthorDate: Wed Feb 14 05:55:31 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 14 06:04:22 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=038ad102
BuildLogger: Fix portage.locks._open_fds memory leak
The _file_close_wrapper __getattribute__ method needs to
be overridden to expose its close method, otherwise the
underlying file's close method is called and the closed
file object remains as a memory leak in the global
portage.locks._open_fds dict. For reference, see similar
classes like portage.util.atomic_ofstream which overrides
methods in the same way.
Bug: https://bugs.gentoo.org/919072
Fixes: df212738bbb2 ("BuildLogger: Close self._stdin after fork")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_async/BuildLogger.py | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/portage/util/_async/BuildLogger.py
b/lib/portage/util/_async/BuildLogger.py
index 9f8a21ab2b..0cfc90a942 100644
--- a/lib/portage/util/_async/BuildLogger.py
+++ b/lib/portage/util/_async/BuildLogger.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Gentoo Authors
+# Copyright 2020-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -31,6 +31,11 @@ class _file_close_wrapper(ObjectProxy):
def _get_target(self):
return object.__getattribute__(self, "_file")
+def __getattribute__(self, attr):
+if attr == "close":
+return object.__getattribute__(self, attr)
+return getattr(object.__getattribute__(self, "_file"), attr)
+
def close(self):
file = object.__getattribute__(self, "_file")
if not file.closed:
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: ba33bc425363977eaf549a087b2469720a79e2a4 Author: Zac Medico gentoo org> AuthorDate: Thu Feb 8 06:46:04 2024 + Commit: Zac Medico gentoo org> CommitDate: Fri Feb 9 08:19:00 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ba33bc42 check_locale: Use multiprocessing.Process instead of os.fork() Since os.fork() is unsafe in threaded processes, use multiprocessing.Process instead. This way the fork will be eliminated when the default multiprocessing start method changes to "spawn". TODO: Make async version of check_locale and call it from EbuildPhase instead of config.environ(), since it's bad to synchronously wait for the process in the main event loop thread where config.environ() tends to be called. Bug: https://bugs.gentoo.org/923841 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/locale.py | 56 +++--- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py index a620dbd544..b5da8d949b 100644 --- a/lib/portage/util/locale.py +++ b/lib/portage/util/locale.py @@ -9,7 +9,8 @@ locale. import locale import logging -import os +import multiprocessing +import sys import textwrap import traceback @@ -96,6 +97,24 @@ def _check_locale(silent): return True +def _set_and_check_locale(silent, env, mylocale): +try: +if env is not None: +try: +locale.setlocale(locale.LC_CTYPE, mylocale) +except locale.Error: +sys.exit(2) + +ret = _check_locale(silent) +if ret is None: +sys.exit(2) +else: +sys.exit(0 if ret else 1) +except Exception: +traceback.print_exc() +sys.exit(2) + + def check_locale(silent=False, env=None): """ Check whether the locale is sane. Returns True if it is, prints @@ -116,29 +135,20 @@ def check_locale(silent=False, env=None): except KeyError: pass -pid = os.fork() -if pid == 0: -try: -if env is not None: -try: -locale.setlocale(locale.LC_CTYPE, portage._native_string(mylocale)) -except locale.Error: -os._exit(2) - -ret = _check_locale(silent) -if ret is None: -os._exit(2) -else: -os._exit(0 if ret else 1) -except Exception: -traceback.print_exc() -os._exit(2) - -pid2, ret = os.waitpid(pid, 0) -assert pid == pid2 +# TODO: Make async version of check_locale and call it from +# EbuildPhase instead of config.environ(), since it's bad to +# synchronously wait for the process in the main event loop +# thread where config.environ() tends to be called. +proc = multiprocessing.Process( +target=_set_and_check_locale, +args=(silent, env, None if env is None else portage._native_string(mylocale)), +) +proc.start() +proc.join() + pyret = None -if os.WIFEXITED(ret): -ret = os.WEXITSTATUS(ret) +if proc.exitcode >= 0: +ret = proc.exitcode if ret != 2: pyret = ret == 0
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/util/, ...
commit: c95fc64abf9698263090b3ffd4a056e989dd2be1
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 9 06:38:41 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 9 08:19:00 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c95fc64a
EbuildPhase: async_check_locale
Change config.environ() check_locale calls to async_check_locale
calls in the EbuildPhase _async_start method in order to eliminate
synchronous waiting for child processes in the main event loop
thread.
Bug: https://bugs.gentoo.org/923841
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildMetadataPhase.py| 21
lib/_emerge/EbuildPhase.py| 28 ++-
lib/portage/package/ebuild/config.py | 26 +++--
lib/portage/util/futures/_asyncio/__init__.py | 9 +
lib/portage/util/locale.py| 28 ++-
5 files changed, 87 insertions(+), 25 deletions(-)
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index f4f685e81c..53b7ad9624 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,12 +8,14 @@ import portage
portage.proxy.lazyimport.lazyimport(
globals(),
+"_emerge.EbuildPhase:_setup_locale",
"portage.package.ebuild._metadata_invalid:eapi_invalid",
)
from portage import os
from portage import _encodings
from portage import _unicode_decode
from portage import _unicode_encode
+from portage.util.futures import asyncio
import fcntl
@@ -44,6 +46,12 @@ class EbuildMetadataPhase(SubProcess):
_files_dict = slot_dict_class(_file_names, prefix="")
def _start(self):
+asyncio.ensure_future(
+self._async_start(), loop=self.scheduler
+).add_done_callback(self._async_start_done)
+
+async def _async_start(self):
+
ebuild_path = self.ebuild_hash.location
with open(
@@ -75,6 +83,9 @@ class EbuildMetadataPhase(SubProcess):
settings.setcpv(self.cpv)
settings.configdict["pkg"]["EAPI"] = parsed_eapi
+# This requires above setcpv and EAPI setup.
+await _setup_locale(self.settings)
+
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
@@ -139,6 +150,16 @@ class EbuildMetadataPhase(SubProcess):
self._proc = retval
+def _async_start_done(self, future):
+future.cancelled() or future.result()
+if future.cancelled():
+self.cancel()
+self._was_cancelled()
+
+if self.returncode is not None:
+self._unregister()
+self.wait()
+
def _output_handler(self):
while True:
buf = self._read_buf(self._files.ebuild)
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index c81bf54a81..c8caf73722 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import (
_prepare_fake_distdir,
_prepare_fake_filesdir,
)
+from portage.eapi import _get_eapi_attrs
from portage.util import writemsg, ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport(
+ "_post_src_install_write_metadata,"
+ "_preinst_bsdflags",
"portage.util.futures.unix_events:_set_nonblocking",
+"portage.util.locale:async_check_locale,split_LC_ALL",
)
from portage import os
from portage import _encodings
from portage import _unicode_encode
+async def _setup_locale(settings):
+eapi_attrs = _get_eapi_attrs(settings["EAPI"])
+if eapi_attrs.posixish_locale:
+split_LC_ALL(settings)
+settings["LC_COLLATE"] = "C"
+# check_locale() returns None when check can not be executed.
+if await async_check_locale(silent=True, env=settings.environ()) is
False:
+# try another locale
+for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
+settings["LC_CTYPE"] = l
+if await async_check_locale(silent=True,
env=settings.environ()):
+# TODO: output the following only once
+# writemsg(
+# _("!!! LC_CTYPE unsupported, using %s instead\n")
+# % self.settings["LC_CTYPE"]
+# )
+break
+else:
+raise AssertionError("C locale did not pass the test!")
+
+
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") +
("_ebuild_lock",)
@@ -94,6 +117,9 @@ class
[gentoo-commits] proj/portage:master commit in: lib/portage/util/elf/, lib/portage/dep/soname/, /
commit: da6607f012572b4d13fdc6fd5ecd4a62d62a417b
Author: Matoro Mahri matoro tk>
AuthorDate: Tue Oct 31 21:00:34 2023 +
Commit: Sam James gentoo org>
CommitDate: Fri Feb 9 07:26:32 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=da6607f0
ELF: add entries for ARC machines
Signed-off-by: Matoro Mahri matoro.tk>
Closes: https://github.com/gentoo/portage/pull/1166
Signed-off-by: Sam James gentoo.org>
NEWS| 1 +
lib/portage/dep/soname/multilib_category.py | 10 ++
lib/portage/util/elf/constants.py | 5 +
3 files changed, 16 insertions(+)
diff --git a/NEWS b/NEWS
index 220a0f4f7e..396723d8a8 100644
--- a/NEWS
+++ b/NEWS
@@ -11,6 +11,7 @@ portage-3.0.62 (UNRELEASED)
Features:
* cnf: make.conf.example.arc: add for the arc arch.
+* ELF: add entries for ARC machines
portage-3.0.61 (2024-01-05)
--
diff --git a/lib/portage/dep/soname/multilib_category.py
b/lib/portage/dep/soname/multilib_category.py
index 14a9eea770..baca439fd2 100644
--- a/lib/portage/dep/soname/multilib_category.py
+++ b/lib/portage/dep/soname/multilib_category.py
@@ -52,6 +52,11 @@ from portage.util.elf.constants import (
EM_AARCH64,
EM_ALPHA,
EM_AMDGPU,
+EM_ARC,
+EM_ARC_COMPACT,
+EM_ARC_COMPACT2,
+EM_ARC_COMPACT3,
+EM_ARC_COMPACT3_64,
EM_ARM,
EM_ALTERA_NIOS2,
EM_IA_64,
@@ -80,6 +85,11 @@ _machine_prefix_map = {
EM_ALPHA: "alpha",
EM_AMDGPU: "amdgpu",
EM_ALTERA_NIOS2: "nios2",
+EM_ARC: "arc",
+EM_ARC_COMPACT: "arc",
+EM_ARC_COMPACT2: "arc",
+EM_ARC_COMPACT3: "arc",
+EM_ARC_COMPACT3_64: "arc",
EM_ARM: "arm",
EM_IA_64: "ia64",
EM_LOONGARCH: "loong",
diff --git a/lib/portage/util/elf/constants.py
b/lib/portage/util/elf/constants.py
index 022e78d776..9216a35353 100644
--- a/lib/portage/util/elf/constants.py
+++ b/lib/portage/util/elf/constants.py
@@ -31,12 +31,17 @@ EM_S390 = 22
EM_ARM = 40
EM_SH = 42
EM_SPARCV9 = 43
+EM_ARC = 45
EM_IA_64 = 50
EM_X86_64 = 62
+EM_ARC_COMPACT = 93
EM_ALTERA_NIOS2 = 113
EM_AARCH64 = 183
+EM_ARC_COMPACT2 = 195
EM_AMDGPU = 224
EM_RISCV = 243
+EM_ARC_COMPACT3_64 = 253
+EM_ARC_COMPACT3 = 255
EM_LOONGARCH = 258
EM_ALPHA = 0x9026
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: fa8e8f1895ed889aece2f67725df55d6ccf127fb Author: Zac Medico gentoo org> AuthorDate: Sat Feb 3 23:41:45 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 7 00:55:46 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa8e8f18 socks5: Migrate to spawn returnproc parameter Bug: https://bugs.gentoo.org/916566 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/socks5.py | 36 +++- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index fedb8599d5..6c68ff4106 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -1,10 +1,9 @@ # SOCKSv5 proxy manager for network-sandbox -# Copyright 2015-2021 Gentoo Authors +# Copyright 2015-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno import os -import signal import socket import portage.data @@ -22,7 +21,8 @@ class ProxyManager: def __init__(self): self.socket_path = None -self._pids = [] +self._proc = None +self._proc_waiter = None def start(self, settings): """ @@ -51,9 +51,9 @@ class ProxyManager: spawn_kwargs.update( uid=portage_uid, gid=portage_gid, groups=userpriv_groups, umask=0o077 ) -self._pids = spawn( +self._proc = spawn( [_python_interpreter, server_bin, self.socket_path], -returnpid=True, +returnproc=True, **spawn_kwargs, ) @@ -61,12 +61,19 @@ class ProxyManager: """ Stop the SOCKSv5 server. """ -for p in self._pids: -os.kill(p, signal.SIGINT) -os.waitpid(p, 0) +if self._proc is not None: +self._proc.terminate() +loop = asyncio.get_event_loop() +if self._proc_waiter is None: +self._proc_waiter = asyncio.ensure_future(self._proc.wait(), loop) +if loop.is_running(): +self._proc_waiter.add_done_callback(lambda future: future.result()) +else: +loop.run_until_complete(self._proc_waiter) self.socket_path = None -self._pids = [] +self._proc = None +self._proc_waiter = None def is_running(self): """ @@ -80,16 +87,11 @@ class ProxyManager: """ Wait for the proxy socket to become ready. This method is a coroutine. """ +if self._proc_waiter is None: +self._proc_waiter = asyncio.ensure_future(self._proc.wait()) while True: -try: -wait_retval = os.waitpid(self._pids[0], os.WNOHANG) -except OSError as e: -if e.errno == errno.EINTR: -continue -raise - -if wait_retval is not None and wait_retval != (0, 0): +if self._proc_waiter.done(): raise OSError(3, "No such process") try:
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/sync/modules/rsync/
commit: 62332ee82b8b88fa5a65aafa7c221ccdaa7d65a8
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 4 00:11:07 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 7 00:55:46 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=62332ee8
RsyncSync: Migrate to spawn returnproc parameter
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/portage/sync/modules/rsync/rsync.py | 40 +--
lib/portage/util/futures/_asyncio/__init__.py | 6 +++-
2 files changed, 24 insertions(+), 22 deletions(-)
diff --git a/lib/portage/sync/modules/rsync/rsync.py
b/lib/portage/sync/modules/rsync/rsync.py
index 175c7f2e8e..5d442d2626 100644
--- a/lib/portage/sync/modules/rsync/rsync.py
+++ b/lib/portage/sync/modules/rsync/rsync.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import datetime
@@ -708,48 +708,47 @@ class RsyncSync(NewBase):
command.append(syncuri.rstrip("/") + "/metadata/timestamp.chk")
command.append(tmpservertimestampfile)
content = None
-pids = []
+proc = None
+proc_waiter = None
+loop = asyncio.get_event_loop()
try:
# Timeout here in case the server is unresponsive. The
# --timeout rsync option doesn't apply to the initial
# connection attempt.
try:
-if self.rsync_initial_timeout:
-
portage.exception.AlarmSignal.register(self.rsync_initial_timeout)
-
-pids.extend(
-portage.process.spawn(command, returnpid=True,
**self.spawn_kwargs)
+proc = portage.process.spawn(
+command, returnproc=True, **self.spawn_kwargs
+)
+proc_waiter = asyncio.ensure_future(proc.wait(), loop)
+future = (
+asyncio.wait_for(
+asyncio.shield(proc_waiter), self.rsync_initial_timeout
+)
+if self.rsync_initial_timeout
+else proc_waiter
)
-exitcode = os.waitpid(pids[0], 0)[1]
+exitcode = loop.run_until_complete(future)
if self.usersync_uid is not None:
portage.util.apply_permissions(
tmpservertimestampfile, uid=os.getuid()
)
content = portage.grabfile(tmpservertimestampfile)
finally:
-if self.rsync_initial_timeout:
-portage.exception.AlarmSignal.unregister()
try:
os.unlink(tmpservertimestampfile)
except OSError:
pass
-except portage.exception.AlarmSignal:
+except (TimeoutError, asyncio.TimeoutError):
# timed out
print("timed out")
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
-if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
-os.kill(pids[0], signal.SIGTERM)
-os.waitpid(pids[0], 0)
+if proc_waiter and not proc_waiter.done():
+proc.terminate()
+loop.run_until_complete(proc_waiter)
# This is the same code rsync uses for timeout.
exitcode = 30
-else:
-if exitcode != os.EX_OK:
-if exitcode & 0xFF:
-exitcode = (exitcode & 0xFF) << 8
-else:
-exitcode = exitcode >> 8
if content:
try:
@@ -758,7 +757,6 @@ class RsyncSync(NewBase):
)
except (OverflowError, ValueError):
pass
-del command, pids, content
if exitcode == os.EX_OK:
if (servertimestamp != 0) and (servertimestamp == timestamp):
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index a5a6cb3a5b..8f1b8e8275 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018-2021 Gentoo Authors
+# Copyright 2018-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = (
@@ -15,9 +15,11 @@ __all__ = (
"set_child_watcher",
"get_event_loop_policy",
"set_event_loop_policy",
+"shield",
"sleep",
"Task",
"wait",
+"wait_for",
)
import types
@@ -33,7 +35,9 @@ from asyncio import (
FIRST_EXCEPTION,
Future,
InvalidStateError,
+shield,
TimeoutError,
+wait_for,
)
import threading
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/executor/, lib/portage/tests/util/futures/
commit: 6c4b542b2a830587869b6180e879b719e97fda66 Author: Zac Medico gentoo org> AuthorDate: Tue Feb 6 04:00:29 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 7 00:49:26 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c4b542b ForkExecutor: multiprocessing spawn compat Use the AsyncFunction create_pipe=False parameter to avoid issues in the pipe code triggered with the "spawn" multiprocessing start method when spawn uses multiprocessing.Process (bug 916566), since these jobs should inherit stdio streams and run in the foreground with no log. Also fix RetryForkExecutorTestCase to avoid pickling issues. Bug: https://bugs.gentoo.org/923854 Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/futures/test_retry.py | 42 ++-- lib/portage/util/futures/executor/fork.py| 6 ++-- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/lib/portage/tests/util/futures/test_retry.py b/lib/portage/tests/util/futures/test_retry.py index a5b56bdc7f..6bd3f4b64a 100644 --- a/lib/portage/tests/util/futures/test_retry.py +++ b/lib/portage/tests/util/futures/test_retry.py @@ -1,4 +1,4 @@ -# Copyright 2018-2023 Gentoo Authors +# Copyright 2018-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from concurrent.futures import Future, ThreadPoolExecutor @@ -9,7 +9,6 @@ import threading import weakref import time -import portage from portage.tests import TestCase from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.backoff import RandomExponentialBackoff @@ -229,16 +228,19 @@ class RetryForkExecutorTestCase(RetryTestCase): @contextlib.contextmanager def _wrap_coroutine_func(self, coroutine_func): +uses_subprocess = isinstance(self._executor, ForkExecutor) parent_loop = global_event_loop() -parent_pid = portage.getpid() pending = weakref.WeakValueDictionary() # Since ThreadPoolExecutor does not propagate cancellation of a # parent_future to the underlying coroutine, use kill_switch to # propagate task cancellation to wrapper, so that HangForever's # thread returns when retry eventually cancels parent_future. -def wrapper(kill_switch): -if portage.getpid() == parent_pid: +if uses_subprocess: +wrapper = _run_coroutine_in_subprocess(coroutine_func) +else: + +def wrapper(kill_switch): # thread in main process def done_callback(result): result.cancelled() or result.exception() or result.result() @@ -262,22 +264,19 @@ class RetryForkExecutorTestCase(RetryTestCase): else: return future.result().result() -# child process -loop = global_event_loop() -try: -return loop.run_until_complete(coroutine_func()) -finally: -loop.close() - def execute_wrapper(): -kill_switch = threading.Event() +# Use kill_switch for threads because they can't be killed +# like processes. Do not pass kill_switch to subprocesses +# because it is not picklable. +kill_switch = None if uses_subprocess else threading.Event() +wrapper_args = [kill_switch] if kill_switch else [] parent_future = asyncio.ensure_future( -parent_loop.run_in_executor(self._executor, wrapper, kill_switch), +parent_loop.run_in_executor(self._executor, wrapper, *wrapper_args), loop=parent_loop, ) def kill_callback(parent_future): -if not kill_switch.is_set(): +if kill_switch is not None and not kill_switch.is_set(): kill_switch.set() parent_future.add_done_callback(kill_callback) @@ -298,6 +297,19 @@ class RetryForkExecutorTestCase(RetryTestCase): future.cancelled() or future.exception() or future.result() +class _run_coroutine_in_subprocess: +def __init__(self, coroutine_func): +self._coroutine_func = coroutine_func + +def __call__(self): +# child process +loop = global_event_loop() +try: +return loop.run_until_complete(self._coroutine_func()) +finally: +loop.close() + + class RetryThreadExecutorTestCase(RetryForkExecutorTestCase): def _setUpExecutor(self): self._executor = ThreadPoolExecutor(max_workers=1) diff --git a/lib/portage/util/futures/executor/fork.py b/lib/portage/util/futures/executor/fork.py index 61ad6aecfb..1e3d010724 100644 --- a/lib/portage/util/futures/executor/fork.py +++ b/lib/portage/util/futures/executor/fork.py @@ -1,4 +1,4 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2024 Gentoo Authors # Distributed under the
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 7d7ef237f3dddcf450fedab5aabfd57d1fb3406d Author: Zac Medico gentoo org> AuthorDate: Tue Feb 6 01:35:25 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 7 00:36:56 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7d7ef237 ForkProcess: Handle BrokenPipeError in _send_fd_pipes Convert _send_fd_pipes BrokenPipeError to asyncio.CancelledError, in order to gracefully handle a concurrently terminated child process as in testAsynchronousLockWaitCancel. Even if the child terminated abnormally, then there is no harm in suppressing the exception here, since the child error should have gone to stderr. Bug: https://bugs.gentoo.org/923852 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_async/ForkProcess.py | 25 + 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/lib/portage/util/_async/ForkProcess.py b/lib/portage/util/_async/ForkProcess.py index cb240d0712..ebcbd94107 100644 --- a/lib/portage/util/_async/ForkProcess.py +++ b/lib/portage/util/_async/ForkProcess.py @@ -153,15 +153,24 @@ class ForkProcess(SpawnProcess): This performs blocking IO, intended for invocation via run_in_executor. """ fd_list = list(set(self._fd_pipes.values())) -self._files.connection.send( -(self._fd_pipes, fd_list), -) -for fd in fd_list: -multiprocessing.reduction.send_handle( -self._files.connection, -fd, -self.pid, +try: +self._files.connection.send( +(self._fd_pipes, fd_list), ) +for fd in fd_list: +multiprocessing.reduction.send_handle( +self._files.connection, +fd, +self.pid, +) +except BrokenPipeError as e: +# This case is triggered by testAsynchronousLockWaitCancel +# when the test case terminates the child process while +# this thread is still sending the fd_pipes (bug 923852). +# Even if the child terminated abnormally, then there is +# no harm in suppressing the exception here, since the +# child error should have gone to stderr. +raise asyncio.CancelledError from e # self._fd_pipes contains duplicates that must be closed. for fd in fd_list:
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 9f96ce5105e7bd2580ae9acc34d6ebad914dae47
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 3 19:36:05 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 3 19:55:19 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9f96ce51
ForkProcess: Use duplicate fd_pipes in _send_fd_pipes thread
In order to allow callers to manage the lifecycle of fd_pipes
file descriptors, create duplicates for _send_fd_pipes to
close when it has finished sending them.
This fixes bug 916601 in a nice way, allowing commit
3b1234ba69a31709cd5aec1ae070901e3a28bb7c to be reverted.
Bug: https://bugs.gentoo.org/916601
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_async/ForkProcess.py | 55 +-
1 file changed, 48 insertions(+), 7 deletions(-)
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index 711bd2a7ba..3acbe34fc6 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -24,6 +24,8 @@ class ForkProcess(SpawnProcess):
"kwargs",
"target",
"_child_connection",
+# Duplicate file descriptors for use by _send_fd_pipes background
thread.
+"_fd_pipes",
)
_file_names = ("connection", "slave_fd")
@@ -53,7 +55,13 @@ class ForkProcess(SpawnProcess):
duplex=self._HAVE_SEND_HANDLE
)
-self._proc = self._spawn(self.args, fd_pipes=self.fd_pipes)
+# Handle fd_pipes in _main instead, since file descriptors are
+# not inherited with the multiprocessing "spawn" start method.
+# Pass fd_pipes=None to spawn here so that it doesn't leave
+# a closed stdin duplicate in fd_pipes (that would trigger
+# "Bad file descriptor" error if we tried to send it via
+# send_handle).
+self._proc = self._spawn(self.args, fd_pipes=None)
self._registered = True
@@ -74,6 +82,25 @@ class ForkProcess(SpawnProcess):
self.fd_pipes[1] = slave_fd
self.fd_pipes[2] = slave_fd
self._files = self._files_dict(connection=connection,
slave_fd=slave_fd)
+
+# Create duplicate file descriptors in self._fd_pipes
+# so that the caller is free to manage the lifecycle
+# of the original fd_pipes.
+self._fd_pipes = {}
+fd_map = {}
+for dest, src in list(self.fd_pipes.items()):
+if src not in fd_map:
+src_new = fd_map[src] = os.dup(src)
+old_fdflags = fcntl.fcntl(src, fcntl.F_GETFD)
+fcntl.fcntl(src_new, fcntl.F_SETFD, old_fdflags)
+os.set_inheritable(
+src_new, not bool(old_fdflags & fcntl.FD_CLOEXEC)
+)
+self._fd_pipes[dest] = fd_map[src]
+
+asyncio.ensure_future(
+self._proc.wait(), self.scheduler
+).add_done_callback(self._close_fd_pipes)
else:
master_fd = connection
@@ -81,6 +108,19 @@ class ForkProcess(SpawnProcess):
master_fd, log_file_path=self.logfile, stdout_fd=stdout_fd
)
+def _close_fd_pipes(self, future):
+"""
+Cleanup self._fd_pipes if needed, since _send_fd_pipes could
+have been cancelled.
+"""
+# future.result() raises asyncio.CancelledError if
+# future.cancelled(), but that should not happen.
+future.result()
+if self._fd_pipes is not None:
+for fd in set(self._fd_pipes.values()):
+os.close(fd)
+self._fd_pipes = None
+
@property
def _fd_pipes_send_handle(self):
"""Returns True if we have a connection to implement fd_pipes via
send_handle."""
@@ -95,9 +135,9 @@ class ForkProcess(SpawnProcess):
Communicate with _bootstrap to send fd_pipes via send_handle.
This performs blocking IO, intended for invocation via run_in_executor.
"""
-fd_list = list(set(self.fd_pipes.values()))
+fd_list = list(set(self._fd_pipes.values()))
self._files.connection.send(
-(self.fd_pipes, fd_list),
+(self._fd_pipes, fd_list),
)
for fd in fd_list:
multiprocessing.reduction.send_handle(
@@ -106,6 +146,11 @@ class ForkProcess(SpawnProcess):
self.pid,
)
+# self._fd_pipes contains duplicates that must be closed.
+for fd in fd_list:
+os.close(fd)
+self._fd_pipes = None
+
async def _main(self, build_logger, pipe_logger, loop=None):
try:
if self._fd_pipes_send_handle:
@@ -167,10 +212,6 @@ class ForkProcess(SpawnProcess):
)
fd_pipes[0] = stdin_dup
-
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/, lib/portage/tests/locks/
commit: d4c495c6bd543d07f3c5d6e72145e177999edaab Author: Zac Medico gentoo org> AuthorDate: Sun Jan 28 23:44:23 2024 + Commit: Zac Medico gentoo org> CommitDate: Mon Jan 29 04:08:42 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d4c495c6 Rely on os.register_at_fork for _ForkWatcher hook invocation Because os.register_at_fork is used instead of multiprocessing.util.register_after_fork since commit cc7c40199850acb3d36f0a6452987231d592c360, we can rely on the _ForkWatcher hook being automatically invoked after os.fork(). Signed-off-by: Zac Medico gentoo.org> lib/portage/process.py| 10 +++--- lib/portage/tests/locks/test_lock_nonblock.py | 3 +-- lib/portage/util/locale.py| 3 +-- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/portage/process.py b/lib/portage/process.py index 28f977a046..0d58adecad 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -1,5 +1,5 @@ # portage.py -- core Portage functionality -# Copyright 1998-2023 Gentoo Authors +# Copyright 1998-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 @@ -456,7 +456,6 @@ def spawn( pid = os.fork() if pid == 0: -portage._ForkWatcher.hook(portage._ForkWatcher) try: _exec( binary, @@ -504,8 +503,8 @@ def spawn( sys.stderr.flush() finally: -# Don't used portage.getpid() here, due to a race with the above -# portage._ForkWatcher cache update. +# Don't used portage.getpid() here, in case there is a race +# with getpid cache invalidation via _ForkWatcher hook. if pid == 0 or (pid is None and _os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier @@ -775,9 +774,6 @@ def _exec( if unshare_pid: main_child_pid = os.fork() if main_child_pid == 0: -# The portage.getpid() cache may need to be updated here, -# in case the pre_exec function invokes portage APIs. -portage._ForkWatcher.hook(portage._ForkWatcher) # pid namespace requires us to become init binary, myargs = ( portage._python_interpreter, diff --git a/lib/portage/tests/locks/test_lock_nonblock.py b/lib/portage/tests/locks/test_lock_nonblock.py index e3f9b4d023..9bb91b428e 100644 --- a/lib/portage/tests/locks/test_lock_nonblock.py +++ b/lib/portage/tests/locks/test_lock_nonblock.py @@ -1,4 +1,4 @@ -# Copyright 2011-2020 Gentoo Authors +# Copyright 2011-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import tempfile @@ -19,7 +19,6 @@ class LockNonblockTestCase(TestCase): lock1 = portage.locks.lockfile(path) pid = os.fork() if pid == 0: -portage._ForkWatcher.hook(portage._ForkWatcher) portage.locks._close_fds() # Disable close_fds since we don't exec # (see _setup_pipes docstring). diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py index 0d0c120157..a620dbd544 100644 --- a/lib/portage/util/locale.py +++ b/lib/portage/util/locale.py @@ -1,4 +1,4 @@ -# Copyright 2015-2020 Gentoo Authors +# Copyright 2015-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 """ @@ -118,7 +118,6 @@ def check_locale(silent=False, env=None): pid = os.fork() if pid == 0: -portage._ForkWatcher.hook(portage._ForkWatcher) try: if env is not None: try:
[gentoo-commits] proj/portage:master commit in: lib/portage/util/file_copy/, lib/_emerge/, lib/portage/util/_async/, ...
commit: 3d55e159c473075c7b2f87c92293b0df6fa57563
Author: Zac Medico gentoo org>
AuthorDate: Sun Jan 28 22:01:58 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Jan 28 23:09:26 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d55e159
*/*: rerun black w/ 24.1.0
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/AsynchronousTask.py | 6 +--
lib/_emerge/EbuildMetadataPhase.py | 11 ++---
lib/_emerge/MergeListItem.py| 1 -
lib/_emerge/PipeReader.py | 1 -
lib/_emerge/SpawnProcess.py | 1 -
lib/_emerge/depgraph.py | 59 ++---
lib/_emerge/resolver/slot_collision.py | 12 ++---
lib/portage/_emirrordist/DeletionIterator.py| 8 ++--
lib/portage/_emirrordist/FetchIterator.py | 8 ++--
lib/portage/_sets/libs.py | 1 -
lib/portage/dbapi/porttree.py | 1 -
lib/portage/dbapi/vartree.py| 28 +++-
lib/portage/dep/__init__.py | 1 -
lib/portage/package/ebuild/config.py| 38
lib/portage/package/ebuild/doebuild.py | 16 ---
lib/portage/proxy/objectproxy.py| 1 -
lib/portage/tests/ebuild/test_fetch.py | 10 ++---
lib/portage/tests/process/test_AsyncFunction.py | 8 ++--
lib/portage/util/_async/PipeLogger.py | 1 -
lib/portage/util/_async/TaskScheduler.py| 1 -
lib/portage/util/_dyn_libs/LinkageMapELF.py | 3 --
lib/portage/util/file_copy/__init__.py | 7 +--
lib/portage/util/futures/_sync_decorator.py | 8 ++--
23 files changed, 119 insertions(+), 112 deletions(-)
diff --git a/lib/_emerge/AsynchronousTask.py b/lib/_emerge/AsynchronousTask.py
index 4290eede36..4049ba5eb6 100644
--- a/lib/_emerge/AsynchronousTask.py
+++ b/lib/_emerge/AsynchronousTask.py
@@ -46,9 +46,9 @@ class AsynchronousTask(SlotObject):
)
self.addExitListener(exit_listener)
waiter.add_done_callback(
-lambda waiter: self.removeExitListener(exit_listener)
-if waiter.cancelled()
-else None
+lambda waiter: (
+self.removeExitListener(exit_listener) if waiter.cancelled()
else None
+)
)
if self.returncode is not None:
# If the returncode is not None, it means the exit event has
already
diff --git a/lib/_emerge/EbuildMetadataPhase.py
b/lib/_emerge/EbuildMetadataPhase.py
index fd695e0253..8905a058fc 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -19,7 +19,6 @@ import fcntl
class EbuildMetadataPhase(SubProcess):
-
"""
Asynchronous interface for the ebuild "depend" phase which is
used to extract metadata from the ebuild.
@@ -200,12 +199,10 @@ class EbuildMetadataPhase(SubProcess):
# entries for unsupported EAPIs.
if self.eapi_supported:
if metadata.get("INHERITED", False):
-metadata[
-"_eclasses_"
-] = self.portdb.repositories.get_repo_for_location(
-self.repo_path
-).eclass_db.get_eclass_data(
-metadata["INHERITED"].split()
+metadata["_eclasses_"] = (
+self.portdb.repositories.get_repo_for_location(
+self.repo_path
+
).eclass_db.get_eclass_data(metadata["INHERITED"].split())
)
else:
metadata["_eclasses_"] = {}
diff --git a/lib/_emerge/MergeListItem.py b/lib/_emerge/MergeListItem.py
index efe485c2e0..ae894704a4 100644
--- a/lib/_emerge/MergeListItem.py
+++ b/lib/_emerge/MergeListItem.py
@@ -13,7 +13,6 @@ from _emerge.PackageUninstall import PackageUninstall
class MergeListItem(CompositeTask):
-
"""
TODO: For parallel scheduling, everything here needs asynchronous
execution support (start, poll, and wait methods).
diff --git a/lib/_emerge/PipeReader.py b/lib/_emerge/PipeReader.py
index 026346e0bb..76ab7f1882 100644
--- a/lib/_emerge/PipeReader.py
+++ b/lib/_emerge/PipeReader.py
@@ -8,7 +8,6 @@ from _emerge.AbstractPollTask import AbstractPollTask
class PipeReader(AbstractPollTask):
-
"""
Reads output from one or more files and saves it in memory,
for retrieval via the getvalue() method. This is driven by
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 72fa72c613..40740df9aa 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -16,7 +16,6 @@ from portage.util.futures import asyncio
class SpawnProcess(SubProcess):
-
"""
Constructor keyword args are passed into
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_dyn_libs/, lib/portage/tests/util/dyn_libs/
commit: 95d3e5e80ab9561db870858c2caf6e3bffbf47b0 Author: Zac Medico gentoo org> AuthorDate: Mon Jan 15 20:30:57 2024 + Commit: Zac Medico gentoo org> CommitDate: Sat Jan 20 05:18:12 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=95d3e5e8 installed_dynlibs: Resolve *.so symlinks Resolve *.so symlinks to check if they point to regular files inside the top directory. If a symlink points outside the top directory then try to follow the corresponding file inside the top directory if it exists, and otherwise stop following. Bug: https://bugs.gentoo.org/921170 Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/dyn_libs/meson.build| 1 + .../tests/util/dyn_libs/test_installed_dynlibs.py | 65 ++ lib/portage/util/_dyn_libs/dyn_libs.py | 43 +- 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/lib/portage/tests/util/dyn_libs/meson.build b/lib/portage/tests/util/dyn_libs/meson.build index ddb08f5b1a..8f2c919c13 100644 --- a/lib/portage/tests/util/dyn_libs/meson.build +++ b/lib/portage/tests/util/dyn_libs/meson.build @@ -1,5 +1,6 @@ py.install_sources( [ +'test_installed_dynlibs.py', 'test_soname_deps.py', '__init__.py', '__test__.py', diff --git a/lib/portage/tests/util/dyn_libs/test_installed_dynlibs.py b/lib/portage/tests/util/dyn_libs/test_installed_dynlibs.py new file mode 100644 index 00..421dcf6061 --- /dev/null +++ b/lib/portage/tests/util/dyn_libs/test_installed_dynlibs.py @@ -0,0 +1,65 @@ +# Copyright 2024 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import os +import tempfile + +from portage.const import BASH_BINARY +from portage.tests import TestCase +from portage.util import ensure_dirs +from portage.util._dyn_libs.dyn_libs import installed_dynlibs +from portage.util.file_copy import copyfile + + +class InstalledDynlibsTestCase(TestCase): +def testInstalledDynlibsRegular(self): +""" +Return True for *.so regular files. +""" +with tempfile.TemporaryDirectory() as directory: +bash_copy = os.path.join(directory, "lib", "libfoo.so") +ensure_dirs(os.path.dirname(bash_copy)) +copyfile(BASH_BINARY, bash_copy) +self.assertTrue(installed_dynlibs(directory)) + +def testInstalledDynlibsOnlySymlink(self): +""" +If a *.so symlink is installed but does not point to a regular +file inside the top directory, installed_dynlibs should return +False (bug 921170). +""" +with tempfile.TemporaryDirectory() as directory: +symlink_path = os.path.join(directory, "lib", "libfoo.so") +ensure_dirs(os.path.dirname(symlink_path)) +os.symlink(BASH_BINARY, symlink_path) +self.assertFalse(installed_dynlibs(directory)) + +def testInstalledDynlibsSymlink(self): +""" +Return True for a *.so symlink pointing to a regular file inside +the top directory. +""" +with tempfile.TemporaryDirectory() as directory: +bash_copy = os.path.join(directory, BASH_BINARY.lstrip(os.sep)) +ensure_dirs(os.path.dirname(bash_copy)) +copyfile(BASH_BINARY, bash_copy) +symlink_path = os.path.join(directory, "lib", "libfoo.so") +ensure_dirs(os.path.dirname(symlink_path)) +os.symlink(bash_copy, symlink_path) +self.assertTrue(installed_dynlibs(directory)) + +def testInstalledDynlibsAbsoluteSymlink(self): +""" +If a *.so symlink target is outside of the top directory, +traversal follows the corresponding file inside the top +directory if it exists, and otherwise stops following the +symlink. +""" +with tempfile.TemporaryDirectory() as directory: +bash_copy = os.path.join(directory, BASH_BINARY.lstrip(os.sep)) +ensure_dirs(os.path.dirname(bash_copy)) +copyfile(BASH_BINARY, bash_copy) +symlink_path = os.path.join(directory, "lib", "libfoo.so") +ensure_dirs(os.path.dirname(symlink_path)) +os.symlink(BASH_BINARY, symlink_path) +self.assertTrue(installed_dynlibs(directory)) diff --git a/lib/portage/util/_dyn_libs/dyn_libs.py b/lib/portage/util/_dyn_libs/dyn_libs.py index ee28e8839c..6f8a07d70d 100644 --- a/lib/portage/util/_dyn_libs/dyn_libs.py +++ b/lib/portage/util/_dyn_libs/dyn_libs.py @@ -1,14 +1,51 @@ -# Copyright 2021 Gentoo Authors +# Copyright 2021-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os +import stat + +import portage def installed_dynlibs(directory): -for _dirpath, _dirnames, filenames in os.walk(directory): +""" +This traverses installed *.so symlinks to check if they point to +regular files. If a symlink
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: df212738bbb209356472911cda79902f0e25918e
Author: Zac Medico gentoo org>
AuthorDate: Tue Dec 5 04:23:56 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Dec 6 16:22:56 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=df212738
BuildLogger: Close self._stdin after fork
In order to ensure that we can observe EOF on the read end of the pipe,
close self._stdin after fork. Since portage.locks._close_fds() already
does something similar for _lock_manager instances which have a close()
method, it will also work with these _file_close_wrapper objects.
The portage.locks._close_fds() function calls close after fork, in
the ForkProcess._bootstrap method. For more general fork coverage,
we could move the _close_fds() call to the _ForkWatcher.hook method
in portage/__init__.py, but I've reserved that for a later change
since _close_fds() has been working fine for us where we call it now.
Bug: https://bugs.gentoo.org/919072
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_async/BuildLogger.py | 34 +++---
1 file changed, 31 insertions(+), 3 deletions(-)
diff --git a/lib/portage/util/_async/BuildLogger.py
b/lib/portage/util/_async/BuildLogger.py
index 502b3390e5..9f8a21ab2b 100644
--- a/lib/portage/util/_async/BuildLogger.py
+++ b/lib/portage/util/_async/BuildLogger.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2021 Gentoo Authors
+# Copyright 2020-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -6,13 +6,41 @@ import subprocess
from _emerge.AsynchronousTask import AsynchronousTask
+import portage
from portage import os
+from portage.proxy.objectproxy import ObjectProxy
from portage.util import shlex_split
from portage.util._async.PipeLogger import PipeLogger
from portage.util._async.PopenProcess import PopenProcess
from portage.util.futures import asyncio
+class _file_close_wrapper(ObjectProxy):
+"""
+Prevent fd inheritance via fork, ensuring that we can observe
+EOF on the read end of the pipe (bug 919072).
+"""
+
+__slots__ = ("_file",)
+
+def __init__(self, file):
+ObjectProxy.__init__(self)
+object.__setattr__(self, "_file", file)
+portage.locks._open_fds[file.fileno()] = self
+
+def _get_target(self):
+return object.__getattribute__(self, "_file")
+
+def close(self):
+file = object.__getattribute__(self, "_file")
+if not file.closed:
+# This must only be called if the file is open,
+# which ensures that file.fileno() does not
+# collide with an open lock file descriptor.
+del portage.locks._open_fds[file.fileno()]
+file.close()
+
+
class BuildLogger(AsynchronousTask):
"""
Write to a log file, with compression support provided by PipeLogger.
@@ -67,7 +95,7 @@ class BuildLogger(AsynchronousTask):
os.close(log_input)
os.close(filter_output)
else:
-self._stdin = os.fdopen(stdin, "wb", 0)
+self._stdin = _file_close_wrapper(os.fdopen(stdin, "wb",
0))
os.close(filter_input)
os.close(filter_output)
@@ -76,7 +104,7 @@ class BuildLogger(AsynchronousTask):
# that is missing or broken somehow, create a pipe that
# logs directly to pipe_logger.
log_input, stdin = os.pipe()
-self._stdin = os.fdopen(stdin, "wb", 0)
+self._stdin = _file_close_wrapper(os.fdopen(stdin, "wb", 0))
# Set background=True so that pipe_logger does not log to stdout.
pipe_logger = PipeLogger(
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/, lib/portage/tests/process/
commit: 6abc969109754ab086db2bac5be1029de1a015c3
Author: Zac Medico gentoo org>
AuthorDate: Fri Oct 20 04:11:48 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Oct 22 04:17:48 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6abc9691
ForkProcess: Implement fd_pipes via send_handle
This new fd_pipes implementation is only enabled
when the multiprocessing start method is not fork,
ensuring backward compatibility with existing
ForkProcess callers that rely on the fork start
method.
Note that the new fd_pipes implementation uses a
thread via run_in_executor, and threads are not
recommended for mixing with the fork start method
due to cpython issue 84559.
Bug: https://bugs.gentoo.org/915896
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/process/test_ForkProcess.py | 7 ++
lib/portage/util/_async/ForkProcess.py| 142 +-
2 files changed, 124 insertions(+), 25 deletions(-)
diff --git a/lib/portage/tests/process/test_ForkProcess.py
b/lib/portage/tests/process/test_ForkProcess.py
index c07c60e9c6..bc0b836f11 100644
--- a/lib/portage/tests/process/test_ForkProcess.py
+++ b/lib/portage/tests/process/test_ForkProcess.py
@@ -4,6 +4,7 @@
import functools
import multiprocessing
import tempfile
+from unittest.mock import patch
from portage import os
from portage.tests import TestCase
@@ -37,3 +38,9 @@ class ForkProcessTestCase(TestCase):
with open(logfile.name, "rb") as output:
self.assertEqual(output.read(), test_string.encode("utf-8"))
+
+def test_spawn_logfile_no_send_handle(self):
+with patch(
+"portage.util._async.ForkProcess.ForkProcess._HAVE_SEND_HANDLE",
new=False
+):
+self.test_spawn_logfile()
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index 09e40a2d3e..6d216a5c43 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -10,6 +10,7 @@ import sys
import portage
from portage import os
+from portage.cache.mappings import slot_dict_class
from portage.util.futures import asyncio
from _emerge.SpawnProcess import SpawnProcess
@@ -26,29 +27,36 @@ class ForkProcess(SpawnProcess):
"_proc_join_task",
)
+_file_names = ("connection", "slave_fd")
+_files_dict = slot_dict_class(_file_names, prefix="")
+
# Number of seconds between poll attempts for process exit status
# (after the sentinel has become ready).
_proc_join_interval = 0.1
-def _start(self):
-if self.fd_pipes or self.logfile:
-if self.fd_pipes:
-if multiprocessing.get_start_method() != "fork":
-raise NotImplementedError(
-'fd_pipes only supported with multiprocessing start
method "fork"'
-)
-super()._start()
-return
+_HAVE_SEND_HANDLE = getattr(multiprocessing.reduction, "HAVE_SEND_HANDLE",
False)
-if self.logfile:
-if multiprocessing.get_start_method() == "fork":
-# Use superclass pty support.
-super()._start()
-return
+def _start(self):
+if multiprocessing.get_start_method() == "fork":
+# Backward compatibility mode.
+super()._start()
+return
+
+# This mode supports multiprocessing start methods
+# other than fork. Note that the fd_pipes implementation
+# uses a thread via run_in_executor, and threads are not
+# recommended for mixing with the fork start method due
+# to cpython issue 84559.
+if self.fd_pipes and not self._HAVE_SEND_HANDLE:
+raise NotImplementedError(
+'fd_pipes only supported with HAVE_SEND_HANDLE or
multiprocessing start method "fork"'
+)
-# Log via multiprocessing.Pipe if necessary.
-pr, pw = multiprocessing.Pipe(duplex=False)
-self._child_connection = pw
+if self.fd_pipes or self.logfile:
+# Log via multiprocessing.Pipe if necessary.
+connection, self._child_connection = multiprocessing.Pipe(
+duplex=self._HAVE_SEND_HANDLE
+)
retval = self._spawn(self.args, fd_pipes=self.fd_pipes)
@@ -59,11 +67,71 @@ class ForkProcess(SpawnProcess):
self._async_waitpid()
else:
self._child_connection.close()
+self.fd_pipes = self.fd_pipes or {}
stdout_fd = None
if not self.background:
-stdout_fd = os.dup(sys.__stdout__.fileno())
+self.fd_pipes.setdefault(0, portage._get_stdin().fileno())
+self.fd_pipes.setdefault(1, sys.__stdout__.fileno())
+self.fd_pipes.setdefault(2, sys.__stderr__.fileno())
+
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/
commit: c98240fba21b319e27fe595d70c671c9ddfb7fd0 Author: Zac Medico gentoo org> AuthorDate: Thu Oct 5 08:20:14 2023 + Commit: Zac Medico gentoo org> CommitDate: Thu Oct 5 08:21:47 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c98240fb AsyncioEventLoop: Put back _ChildWatcherThreadSafetyWrapper In case there are multiple loops running in different threads, use _ChildWatcherThreadSafetyWrapper. This partially reverts commit 690ac6e78c4099e83b84ef11c0b4064b077a8ef0. Bug: https://bugs.gentoo.org/914873 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_eventloop/asyncio_event_loop.py | 30 ++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index fe941b420a..2b21e6dfaf 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -112,7 +112,7 @@ class AsyncioEventLoop(_AbstractEventLoop): watcher = ThreadedChildWatcher() watcher.attach_loop(self._loop) -self._child_watcher = watcher +self._child_watcher = _ChildWatcherThreadSafetyWrapper(self, watcher) return self._child_watcher @@ -153,3 +153,31 @@ class AsyncioEventLoop(_AbstractEventLoop): except ValueError: # This is intended to fail when not called in the main thread. pass + + +class _ChildWatcherThreadSafetyWrapper: +""" +This class provides safety if multiple loops are running in different threads. +""" + +def __init__(self, loop, real_watcher): +self._loop = loop +self._real_watcher = real_watcher + +def close(self): +pass + +def __enter__(self): +return self + +def __exit__(self, a, b, c): +pass + +def _child_exit(self, pid, status, callback, *args): +self._loop.call_soon_threadsafe(callback, pid, status, *args) + +def add_child_handler(self, pid, callback, *args): +self._real_watcher.add_child_handler(pid, self._child_exit, callback, *args) + +def remove_child_handler(self, pid): +return self._real_watcher.remove_child_handler(pid)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/
commit: 690ac6e78c4099e83b84ef11c0b4064b077a8ef0 Author: Zac Medico gentoo org> AuthorDate: Thu Oct 5 06:27:27 2023 + Commit: Zac Medico gentoo org> CommitDate: Thu Oct 5 06:57:11 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=690ac6e7 Eliminate deprecated asyncio.get_child_watcher() usage Use PidfdChildWatcher if os.pidfd_open is available and works, and otherwise use ThreadedChildWatcher which should work in any case. The _ChildWatcherThreadSafetyWrapper class is not needed because both PidfdChildWatcher and ThreadedChildWatcher use the running event loop to invoke the child handler callback, and it is safe to assume that the current AsyncioEventLoop instance is the running loop when it is used to obtain a child watcher instance. Bug: https://bugs.gentoo.org/914873 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_eventloop/asyncio_event_loop.py | 51 +++ 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index 37b5b2706c..fe941b420a 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -6,6 +6,12 @@ import signal import asyncio as _real_asyncio from asyncio.events import AbstractEventLoop as _AbstractEventLoop +from asyncio.unix_events import ThreadedChildWatcher + +try: +from asyncio.unix_events import PidfdChildWatcher +except ImportError: +PidfdChildWatcher = None import portage @@ -90,9 +96,24 @@ class AsyncioEventLoop(_AbstractEventLoop): @return: the internal event loop's AbstractChildWatcher interface """ if self._child_watcher is None: -self._child_watcher = _ChildWatcherThreadSafetyWrapper( -self, _real_asyncio.get_child_watcher() -) +pidfd_works = False +if PidfdChildWatcher is not None and hasattr(os, "pidfd_open"): +try: +fd = os.pidfd_open(portage.getpid()) +except Exception: +pass +else: +os.close(fd) +pidfd_works = True + +if pidfd_works: +watcher = PidfdChildWatcher() +else: +watcher = ThreadedChildWatcher() + +watcher.attach_loop(self._loop) +self._child_watcher = watcher + return self._child_watcher @property @@ -132,27 +153,3 @@ class AsyncioEventLoop(_AbstractEventLoop): except ValueError: # This is intended to fail when not called in the main thread. pass - - -class _ChildWatcherThreadSafetyWrapper: -def __init__(self, loop, real_watcher): -self._loop = loop -self._real_watcher = real_watcher - -def close(self): -pass - -def __enter__(self): -return self - -def __exit__(self, a, b, c): -pass - -def _child_exit(self, pid, status, callback, *args): -self._loop.call_soon_threadsafe(callback, pid, status, *args) - -def add_child_handler(self, pid, callback, *args): -self._real_watcher.add_child_handler(pid, self._child_exit, callback, *args) - -def remove_child_handler(self, pid): -return self._real_watcher.remove_child_handler(pid)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: ed458fa634c37c13cadd436b38498678f4ee103d Author: Zac Medico gentoo org> AuthorDate: Thu Oct 5 05:54:40 2023 + Commit: Zac Medico gentoo org> CommitDate: Thu Oct 5 05:59:39 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ed458fa6 ForkProcess: Warn if _run method is used Bug: https://bugs.gentoo.org/915099 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_async/ForkProcess.py | 6 ++ 1 file changed, 6 insertions(+) diff --git a/lib/portage/util/_async/ForkProcess.py b/lib/portage/util/_async/ForkProcess.py index 3deaf18fd0..1d2d220ed4 100644 --- a/lib/portage/util/_async/ForkProcess.py +++ b/lib/portage/util/_async/ForkProcess.py @@ -4,6 +4,7 @@ import fcntl import functools import multiprocessing +import warnings import signal import sys @@ -46,6 +47,11 @@ class ForkProcess(SpawnProcess): target = self._run args = None kwargs = None +warnings.warn( +'portage.util._async.ForkProcess.ForkProcess._run is deprecated in favor of the "target" parameter', +UserWarning, +stacklevel=2, +) # Since multiprocessing.Process closes sys.__stdin__, create a # temporary duplicate of fd_pipes[0] so that sys.__stdin__ can
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 73eb44fce683a8cbfca195f01783b94eea6e7eca Author: Zac Medico gentoo org> AuthorDate: Wed Oct 4 03:42:41 2023 + Commit: Zac Medico gentoo org> CommitDate: Wed Oct 4 03:43:18 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=73eb44fc FileCopier: multiprocessing spawn compat Bug: https://bugs.gentoo.org/915099 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_async/FileCopier.py | 18 +++--- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/portage/util/_async/FileCopier.py b/lib/portage/util/_async/FileCopier.py index da0e85ee4e..d53ff08591 100644 --- a/lib/portage/util/_async/FileCopier.py +++ b/lib/portage/util/_async/FileCopier.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Gentoo Authors +# Copyright 2013-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os as _os @@ -20,16 +20,20 @@ class FileCopier(AsyncTaskFuture): def _start(self): self.future = asyncio.ensure_future( -self.scheduler.run_in_executor(ForkExecutor(loop=self.scheduler), self._run) +self.scheduler.run_in_executor( +ForkExecutor(loop=self.scheduler), +self._target, +self.src_path, +self.dest_path, +) ) super()._start() -def _run(self): -src_path = _unicode_encode( -self.src_path, encoding=_encodings["fs"], errors="strict" -) +@staticmethod +def _target(src_path, dest_path): +src_path = _unicode_encode(src_path, encoding=_encodings["fs"], errors="strict") dest_path = _unicode_encode( -self.dest_path, encoding=_encodings["fs"], errors="strict" +dest_path, encoding=_encodings["fs"], errors="strict" ) copyfile(src_path, dest_path) apply_stat_permissions(dest_path, _os.stat(src_path))
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 3dfe5e326eaaca877bc7a45ec84d6b39fc8d137a
Author: Zac Medico gentoo org>
AuthorDate: Wed Oct 4 02:56:10 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Oct 4 03:07:12 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3dfe5e32
FileDigester: Migrate to AsyncFunction
Bug: https://bugs.gentoo.org/915099
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_async/FileDigester.py | 71 +
1 file changed, 11 insertions(+), 60 deletions(-)
diff --git a/lib/portage/util/_async/FileDigester.py
b/lib/portage/util/_async/FileDigester.py
index ce334ee95a..6491423ae4 100644
--- a/lib/portage/util/_async/FileDigester.py
+++ b/lib/portage/util/_async/FileDigester.py
@@ -1,13 +1,13 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-from portage import os
+import functools
+
from portage.checksum import perform_multiple_checksums
-from portage.util._async.ForkProcess import ForkProcess
-from _emerge.PipeReader import PipeReader
+from portage.util._async.AsyncFunction import AsyncFunction
-class FileDigester(ForkProcess):
+class FileDigester(AsyncFunction):
"""
Asynchronously generate file digests. Pass in file_path and
hash_names, and after successful execution, the digests
@@ -17,64 +17,15 @@ class FileDigester(ForkProcess):
__slots__ = (
"file_path",
-"digests",
"hash_names",
-"_digest_pipe_reader",
-"_digest_pw",
)
def _start(self):
-pr, pw = os.pipe()
-self.fd_pipes = {}
-self.fd_pipes[pw] = pw
-self._digest_pw = pw
-self._digest_pipe_reader = PipeReader(
-input_files={"input": pr}, scheduler=self.scheduler
+self.target = functools.partial(
+perform_multiple_checksums, self.file_path, hashes=self.hash_names
)
-self._digest_pipe_reader.addExitListener(self._digest_pipe_reader_exit)
-self._digest_pipe_reader.start()
-ForkProcess._start(self)
-os.close(pw)
-
-def _run(self):
-digests = perform_multiple_checksums(self.file_path,
hashes=self.hash_names)
-
-buf = "".join("%s=%s\n" % item for item in
digests.items()).encode("utf_8")
-
-while buf:
-buf = buf[os.write(self._digest_pw, buf) :]
-
-return os.EX_OK
-
-def _parse_digests(self, data):
-digests = {}
-for line in data.decode("utf_8").splitlines():
-parts = line.split("=", 1)
-if len(parts) == 2:
-digests[parts[0]] = parts[1]
-
-self.digests = digests
-
-def _async_waitpid(self):
-# Ignore this event, since we want to ensure that we
-# exit only after _digest_pipe_reader has reached EOF.
-if self._digest_pipe_reader is None:
-ForkProcess._async_waitpid(self)
-
-def _digest_pipe_reader_exit(self, pipe_reader):
-self._parse_digests(pipe_reader.getvalue())
-self._digest_pipe_reader = None
-if self.pid is None:
-self._unregister()
-self._async_wait()
-else:
-self._async_waitpid()
-
-def _unregister(self):
-ForkProcess._unregister(self)
+super()._start()
-pipe_reader = self._digest_pipe_reader
-if pipe_reader is not None:
-self._digest_pipe_reader = None
-pipe_reader.removeExitListener(self._digest_pipe_reader_exit)
-pipe_reader.cancel()
+@property
+def digests(self):
+return self.result
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 72760ab948047cf1f5b9e1c1fcf9d2c4934afc8c
Author: Zac Medico gentoo org>
AuthorDate: Tue Oct 3 06:24:21 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Oct 3 14:47:37 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=72760ab9
ForkProcess: Add target constructor parameter
If the _run method is not implemented, then use a new
"target" constructor parameter like multiprocessing.Process.
Support "args" and "kwargs" constructor parameters as well.
If the _run method is implemented, then behave in a backward
compatible manner, and ignore the "args" and "kwargs" constructor
parameters which are used by the AsyncFunction subclass.
Bug: https://bugs.gentoo.org/915099
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_async/AsyncFunction.py | 7 +--
lib/portage/util/_async/ForkProcess.py | 35 +++-
2 files changed, 31 insertions(+), 11 deletions(-)
diff --git a/lib/portage/util/_async/AsyncFunction.py
b/lib/portage/util/_async/AsyncFunction.py
index 8c13b3f5ba..f27b255a55 100644
--- a/lib/portage/util/_async/AsyncFunction.py
+++ b/lib/portage/util/_async/AsyncFunction.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2020 Gentoo Authors
+# Copyright 2015-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import pickle
@@ -16,13 +16,8 @@ class AsyncFunction(ForkProcess):
"result" attribute after the forked process has exited.
"""
-# NOTE: This class overrides the meaning of the SpawnProcess 'args'
-# attribute, and uses it to hold the positional arguments for the
-# 'target' function.
__slots__ = (
-"kwargs",
"result",
-"target",
"_async_func_reader",
"_async_func_reader_pw",
)
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index 22a0e0cd85..3deaf18fd0 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2021 Gentoo Authors
+# Copyright 2012-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import fcntl
@@ -14,7 +14,15 @@ from _emerge.SpawnProcess import SpawnProcess
class ForkProcess(SpawnProcess):
-__slots__ = ("_proc", "_proc_join_task")
+# NOTE: This class overrides the meaning of the SpawnProcess 'args'
+# attribute, and uses it to hold the positional arguments for the
+# 'target' function.
+__slots__ = (
+"kwargs",
+"target",
+"_proc",
+"_proc_join_task",
+)
# Number of seconds between poll attempts for process exit status
# (after the sentinel has become ready).
@@ -27,6 +35,18 @@ class ForkProcess(SpawnProcess):
any pre-fork and post-fork interpreter housekeeping that it provides,
promoting a healthy state for the forked interpreter.
"""
+
+if self.__class__._run is ForkProcess._run:
+# target replaces the deprecated self._run method
+target = self.target
+args = self.args
+kwargs = self.kwargs
+else:
+# _run implementation triggers backward-compatibility mode
+target = self._run
+args = None
+kwargs = None
+
# Since multiprocessing.Process closes sys.__stdin__, create a
# temporary duplicate of fd_pipes[0] so that sys.__stdin__ can
# be restored in the subprocess, in case this is needed for
@@ -41,7 +61,8 @@ class ForkProcess(SpawnProcess):
)
fd_pipes[0] = stdin_dup
self._proc = multiprocessing.Process(
-target=self._bootstrap, args=(fd_pipes,)
+target=self._bootstrap,
+args=(fd_pipes, target, args, kwargs),
)
self._proc.start()
finally:
@@ -122,7 +143,8 @@ class ForkProcess(SpawnProcess):
self._proc_join_task.cancel()
self._proc_join_task = None
-def _bootstrap(self, fd_pipes):
+@staticmethod
+def _bootstrap(fd_pipes, target, args, kwargs):
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
@@ -159,7 +181,10 @@ class ForkProcess(SpawnProcess):
)
sys.__stdin__ = sys.stdin
-sys.exit(self._run())
+sys.exit(target(*(args or []), **(kwargs or {})))
def _run(self):
+"""
+Deprecated and replaced with the "target" constructor parameter.
+"""
raise NotImplementedError(self)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 04184a03b74669fdb48403cb8002de6395cf8684
Author: Zac Medico gentoo org>
AuthorDate: Tue Oct 3 07:33:29 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Oct 3 14:48:01 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=04184a03
AsyncFunction: Migrate to ForkProcess target parameter
Bug: https://bugs.gentoo.org/915099
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_async/AsyncFunction.py | 12 +++-
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/lib/portage/util/_async/AsyncFunction.py
b/lib/portage/util/_async/AsyncFunction.py
index f27b255a55..e13daaebb0 100644
--- a/lib/portage/util/_async/AsyncFunction.py
+++ b/lib/portage/util/_async/AsyncFunction.py
@@ -1,6 +1,7 @@
# Copyright 2015-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import functools
import pickle
import traceback
@@ -19,26 +20,27 @@ class AsyncFunction(ForkProcess):
__slots__ = (
"result",
"_async_func_reader",
-"_async_func_reader_pw",
)
def _start(self):
pr, pw = os.pipe()
self.fd_pipes = {} if self.fd_pipes is None else self.fd_pipes
self.fd_pipes[pw] = pw
-self._async_func_reader_pw = pw
self._async_func_reader = PipeReader(
input_files={"input": pr}, scheduler=self.scheduler
)
self._async_func_reader.addExitListener(self._async_func_reader_exit)
self._async_func_reader.start()
+# args and kwargs are passed as additional args by
ForkProcess._bootstrap.
+self.target = functools.partial(self._target_wrapper, pw, self.target)
ForkProcess._start(self)
os.close(pw)
-def _run(self):
+@staticmethod
+def _target_wrapper(pw, target, *args, **kwargs):
try:
-result = self.target(*(self.args or []), **(self.kwargs or {}))
-os.write(self._async_func_reader_pw, pickle.dumps(result))
+result = target(*args, **kwargs)
+os.write(pw, pickle.dumps(result))
except Exception:
traceback.print_exc()
return 1
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, /
commit: 8008e209d900dc988217ce3721292ba895cd0494
Author: James Le Cuirot gentoo org>
AuthorDate: Sun Oct 1 09:32:33 2023 +
Commit: James Le Cuirot gentoo org>
CommitDate: Mon Oct 2 21:38:18 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8008e209
env-update: Write /usr/etc/ld.so.conf to fix bfd in some obscure cases
This is only needed on prefixed systems. bfd currently reads
${EPREFIX}/etc/ld.so.conf and adds the prefix to these paths, but these
paths are already prefixed. We need them to stay prefixed for ldconfig
and the runtime linker. bfd will use ${EPREFIX}/usr/etc/ld.so.conf
instead if that is present, so we can write the unprefixed paths there.
Other linkers do not use these files at all. We tried to patch bfd to
not use them either, as it shouldn't really be necessary, but that
broke some cases, so we are trying this safer approach instead.
env-update does not write the files under /etc/ld.so.conf.d, but we
shouldn't need to handle these in any case, as all known instances are
not affected by this issue.
Bug: https://bugs.gentoo.org/892549
Closes: https://github.com/gentoo/portage/pull/1105
Signed-off-by: James Le Cuirot gentoo.org>
NEWS | 3 +++
lib/portage/util/env_update.py | 19 +++
2 files changed, 22 insertions(+)
diff --git a/NEWS b/NEWS
index 75680fce18..c6d0247691 100644
--- a/NEWS
+++ b/NEWS
@@ -33,6 +33,9 @@ Bug fixes:
* emerge: fix application count when listing search results for ambiguous
packages (bug #915054).
+* env-update: Write ${EPREFIX}/usr/etc/ld.so.conf with unprefixed paths on
+ prefixed systems to fix the bfd linker in some obscure cases (bug #892549).
+
Cleanups:
* vartree: Remove unused variables and parameters
diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py
index 04fde5a52c..b19a853251 100644
--- a/lib/portage/util/env_update.py
+++ b/lib/portage/util/env_update.py
@@ -221,6 +221,25 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
myfd.write(x + "\n")
myfd.close()
+if eprefix:
+# ldconfig needs ld.so.conf paths to be prefixed, but the bfd
linker
+# needs them unprefixed, so write an alternative ld.so.conf file
for
+# the latter. Other linkers do not use these files. See ldelf.c in
+# binutils for precise bfd behavior, as well as bug #892549.
+ldsoconf_path = os.path.join(eroot, "usr", "etc", "ld.so.conf")
+ensure_dirs(os.path.dirname(ldsoconf_path), mode=0o755)
+myfd = atomic_ofstream(ldsoconf_path)
+myfd.write(
+"# ld.so.conf autogenerated by env-update; make all changes
to\n"
+f"# contents of {eprefix}/etc/env.d directory.\n"
+"# This file is only used by the bfd linker. The paths are
not\n"
+"# prefixed as this is automatically added by the linker.\n"
+)
+for x in specials["LDPATH"]:
+if x.startswith(eprefix + os.path.sep):
+myfd.write(x[len(eprefix) :] + "\n")
+myfd.close()
+
potential_lib_dirs = set()
for lib_dir_glob in ("usr/lib*", "lib*"):
x = os.path.join(eroot, lib_dir_glob)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 7ff8f85350ad3bf35cfdac1c471e399a04220ed1
Author: James Le Cuirot gentoo org>
AuthorDate: Sun Oct 1 09:15:50 2023 +
Commit: James Le Cuirot gentoo org>
CommitDate: Mon Oct 2 21:33:21 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7ff8f853
env_update: Write prefixed paths to files and terminal where appropriate
This change is entirely cosmetic as it only affects comments and
informational messages.
I wasn't sure what to do in the FreeBSD/DragonFly case because
/var/run/ld-elf.so.hints probably isn't prefixed, so I left it alone.
Signed-off-by: James Le Cuirot gentoo.org>
lib/portage/util/env_update.py | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py
index a827963ab4..04fde5a52c 100644
--- a/lib/portage/util/env_update.py
+++ b/lib/portage/util/env_update.py
@@ -216,7 +216,7 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
# ld.so.conf needs updating and ldconfig needs to be run
myfd = atomic_ofstream(ldsoconf_path)
myfd.write("# ld.so.conf autogenerated by env-update; make all changes
to\n")
-myfd.write("# contents of /etc/env.d directory\n")
+myfd.write(f"# contents of {eprefix}/etc/env.d directory.\n")
for x in specials["LDPATH"]:
myfd.write(x + "\n")
myfd.close()
@@ -242,7 +242,7 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
newprelink.write(
"# prelink.conf autogenerated by env-update; make all changes to\n"
)
-newprelink.write("# contents of /etc/env.d directory\n")
+newprelink.write(f"# contents of {eprefix}/etc/env.d directory\n")
for x in sorted(potential_lib_dirs) + ["bin", "sbin"]:
newprelink.write(f"-l /{x}\n")
@@ -286,7 +286,7 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
== b"# prelink.conf autogenerated by env-update; make all
changes to\n"
):
f = atomic_ofstream(prelink_conf)
-f.write("-c /etc/prelink.conf.d/*.conf\n")
+f.write(f"-c {eprefix}/etc/prelink.conf.d/*.conf\n")
f.close()
except OSError as e:
if e.errno != errno.ENOENT:
@@ -361,7 +361,7 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
# an older package installed ON TOP of a newer version will cause
ldconfig
# to overwrite the symlinks we just made. -X means no links. After
'clean'
# we can safely create links.
-writemsg_level(_(f">>> Regenerating
{target_root}etc/ld.so.cache...\n"))
+writemsg_level(_(f">>> Regenerating {eroot}etc/ld.so.cache...\n"))
ret = subprocess.run(
[ldconfig, "-X", "-r", target_root], cwd="/"
).returncode
@@ -392,8 +392,8 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
notice += "# DO NOT EDIT THIS FILE."
penvnotice = notice + " CHANGES TO STARTUP PROFILES\n"
cenvnotice = penvnotice[:]
-penvnotice += "# GO INTO /etc/profile NOT /etc/profile.env\n\n"
-cenvnotice += "# GO INTO /etc/csh.cshrc NOT /etc/csh.env\n\n"
+penvnotice += f"# GO INTO {eprefix}/etc/profile NOT
{eprefix}/etc/profile.env\n\n"
+cenvnotice += f"# GO INTO {eprefix}/etc/csh.cshrc NOT
{eprefix}/etc/csh.env\n\n"
# create /etc/profile.env for bash support
profile_env_path = os.path.join(eroot, "etc", "profile.env")
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/
commit: 34e4d5655360d135dbf2cd3a073d07d96e23cf74 Author: Zac Medico gentoo org> AuthorDate: Tue Sep 26 05:09:53 2023 + Commit: Zac Medico gentoo org> CommitDate: Tue Sep 26 05:14:21 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=34e4d565 Remove deprecated AbstractChildWatcher usage Fixes this DeprecationWarning triggered by lib/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py: lib/portage/util/_eventloop/asyncio_event_loop.py:138: DeprecationWarning: 'AbstractChildWatcher' is deprecated as of Python 3.12 and will be removed in Python 3.14. class _ChildWatcherThreadSafetyWrapper(_AbstractChildWatcher): Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_eventloop/asyncio_event_loop.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index 88933af9d2..37b5b2706c 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -1,4 +1,4 @@ -# Copyright 2018-2021 Gentoo Authors +# Copyright 2018-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os @@ -6,7 +6,6 @@ import signal import asyncio as _real_asyncio from asyncio.events import AbstractEventLoop as _AbstractEventLoop -from asyncio.unix_events import AbstractChildWatcher as _AbstractChildWatcher import portage @@ -135,7 +134,7 @@ class AsyncioEventLoop(_AbstractEventLoop): pass -class _ChildWatcherThreadSafetyWrapper(_AbstractChildWatcher): +class _ChildWatcherThreadSafetyWrapper: def __init__(self, loop, real_watcher): self._loop = loop self._real_watcher = real_watcher
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, /, lib/portage/dbapi/
commit: 1d3b22621d332331da6e48ae653e983406f44e5f
Author: gcarq protonmail com>
AuthorDate: Fri Sep 1 16:46:23 2023 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Sep 1 19:13:27 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1d3b2262
emerge: ensure paths are UTF-8 encoded in _needs_move()
Bug: https://bugs.gentoo.org/913103
Closes: https://github.com/gentoo/portage/pull/1086
Signed-off-by: Michael Egger protonmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
NEWS | 2 ++
lib/portage/dbapi/vartree.py | 7 +--
lib/portage/util/movefile.py | 5 +++--
3 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/NEWS b/NEWS
index aa7ffd2652..65e75759bf 100644
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,8 @@ Features:
Bug fixes:
* Prevent gpg from removing /dev/null when unlocking signing key (bug #912808).
+* emerge: ensure paths are UTF-8 encoded in _needs_move() (bug #913103).
+
portage-3.0.51 (2023-08-20)
--
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index dac3508801..7d1bba712e 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -6273,12 +6273,15 @@ class dblink:
if mydmode is None or not stat.S_ISREG(mydmode) or mymode != mydmode:
return True
+src_bytes = _unicode_encode(mysrc, encoding=_encodings["fs"],
errors="strict")
+dest_bytes = _unicode_encode(mydest, encoding=_encodings["fs"],
errors="strict")
+
if "xattr" in self.settings.features:
excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
-if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs):
+if not _cmpxattr(src_bytes, dest_bytes, exclude=excluded_xattrs):
return True
-return not filecmp.cmp(mysrc, mydest, shallow=False)
+return not filecmp.cmp(src_bytes, dest_bytes, shallow=False)
def merge(
diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
index e2f19ba92b..75100a3acd 100644
--- a/lib/portage/util/movefile.py
+++ b/lib/portage/util/movefile.py
@@ -105,10 +105,11 @@ def _copyxattr(src, dest, exclude=None):
)
-def _cmpxattr(src, dest, exclude=None):
+def _cmpxattr(src: bytes, dest: bytes, exclude=None) -> bool:
"""
Compares extended attributes between |src| and |dest| and returns True
-if they are equal or xattrs are not supported, False otherwise
+if they are equal or xattrs are not supported, False otherwise.
+Assumes all given paths are UTF-8 encoded.
"""
try:
src_attrs = xattr.list(src)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, /
commit: 19c27a2471c78d5e17b14325477fee60ead791e5
Author: James Le Cuirot gentoo org>
AuthorDate: Sun Jul 30 13:19:10 2023 +
Commit: Sam James gentoo org>
CommitDate: Wed Aug 2 06:31:20 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=19c27a24
Use the correct library path when launching scripts directly from a venv
It's not clear what bin_entry_point.sh was trying to do before. The
regular expression didn't appear to match any likely shebang. The
wrapper already runs under the desired Python, so we only need to use
sys.executable, which points to the venv's python symlink.
We don't need to worry about handling non-Python scripts any more either
as the new Meson-based build system just installs these directly to bin
rather than creating an entrypoint for them. Any Python-based Portage
scripts they execute are now tried from the same directory first and
will therefore use the correct environment, as above.
Signed-off-by: James Le Cuirot gentoo.org>
Signed-off-by: Sam James gentoo.org>
NEWS| 3 +++
lib/portage/util/bin_entry_point.py | 18 +-
2 files changed, 8 insertions(+), 13 deletions(-)
diff --git a/NEWS b/NEWS
index 0e3541af4..53db165e8 100644
--- a/NEWS
+++ b/NEWS
@@ -20,6 +20,9 @@ Bug fixes:
* Ensure non-Python (s)bin scripts launch other Python-based Portage scripts
using the same environment.
+* Use the correct Python library path when launching scripts directly from a
+ virtual environment.
+
portage-3.0.49 (2023-06-21)
--
diff --git a/lib/portage/util/bin_entry_point.py
b/lib/portage/util/bin_entry_point.py
index bb012b6b7..efa8b17b7 100644
--- a/lib/portage/util/bin_entry_point.py
+++ b/lib/portage/util/bin_entry_point.py
@@ -1,9 +1,8 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2021-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["bin_entry_point"]
-import re
import sys
from portage.const import PORTAGE_BIN_PATH
@@ -18,17 +17,10 @@ def bin_entry_point():
"""
script_path = os.path.join(PORTAGE_BIN_PATH, os.path.basename(sys.argv[0]))
if os.access(script_path, os.X_OK):
-with open(script_path) as f:
-shebang = f.readline()
-python_match = re.search(r"/python[\d\.]*\s+([^/]*)\s+$", shebang)
-if python_match:
-sys.argv = [
-os.path.join(os.path.dirname(sys.argv[0]), "python"),
-python_match.group(1),
-script_path,
-] + sys.argv[1:]
-os.execvp(sys.argv[0], sys.argv)
-sys.argv[0] = script_path
+sys.argv = [
+sys.executable,
+script_path,
+] + sys.argv[1:]
os.execvp(sys.argv[0], sys.argv)
else:
print("File not found:", script_path, file=sys.stderr)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 581f923b9f888efc3313b6e06ba8a3cf84641d75
Author: Mike Gilbert gentoo org>
AuthorDate: Wed Jul 26 14:27:48 2023 +
Commit: Mike Gilbert gentoo org>
CommitDate: Wed Jul 26 14:27:48 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=581f923b
env_update: drop os.waitstatus_to_exitcode
The subprocess.run() function does this translation already.
Fixes: 3bba408e214ae27bdf924ba90ad4b0919a85f3c8
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/env_update.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py
index 9d3b5bb25..a827963ab 100644
--- a/lib/portage/util/env_update.py
+++ b/lib/portage/util/env_update.py
@@ -381,7 +381,6 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
cwd="/",
).returncode
-ret = os.waitstatus_to_exitcode(ret)
if ret > 0:
writemsg(f"!!! ldconfig failed with exit status {ret}\n",
noiselevel=-1)
if ret < 0:
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, /
commit: 3da0ff3d2dd47469becd80e744c0c3ced448a516
Author: Sam James gentoo org>
AuthorDate: Sat Jul 15 08:16:45 2023 +
Commit: Sam James gentoo org>
CommitDate: Wed Jul 26 07:58:30 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3da0ff3d
lib: env_update: check ldconfig exit code
Bug: https://bugs.gentoo.org/910376
Signed-off-by: Sam James gentoo.org>
NEWS | 2 ++
lib/portage/util/env_update.py | 13 ++---
2 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/NEWS b/NEWS
index fcbbdb2a5..4c54ba382 100644
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,8 @@ Bug fixes:
* install-qa-check.d/05prefix: Fix prefixifying shebang for >= EAPI 7 ebuilds
(bug #909147).
+* env-update: Check exit code from ldconfig (bug #910376).
+
* emerge: Fix 'no ebuilds available' message always mentioning binpkgs
(bug #909853).
diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py
index d76042a6b..5c2b2fdd5 100644
--- a/lib/portage/util/env_update.py
+++ b/lib/portage/util/env_update.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["env_update"]
@@ -6,6 +6,7 @@ __all__ = ["env_update"]
import errno
import glob
import stat
+import sys
import time
import portage
@@ -363,12 +364,12 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
writemsg_level(
_(">>> Regenerating %setc/ld.so.cache...\n") % (target_root,)
)
-os.system(f"cd / ; {ldconfig} -X -r '{target_root}'")
+ret = os.system(f"cd / ; {ldconfig} -X -r '{target_root}'")
elif ostype in ("FreeBSD", "DragonFly"):
writemsg_level(
_(">>> Regenerating %svar/run/ld-elf.so.hints...\n") %
target_root
)
-os.system(
+ret = os.system(
(
"cd / ; %s -elf -i "
+ "-f '%svar/run/ld-elf.so.hints' '%setc/ld.so.conf'"
@@ -376,6 +377,12 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
% (ldconfig, target_root, target_root)
)
+ret = os.waitstatus_to_exitcode(ret)
+if ret > 0:
+writemsg(f"!!! ldconfig failed with exit status {ret}\n",
noiselevel=-1)
+if ret < 0:
+writemsg(f"!!! ldconfig was killed with signal {-ret}\n",
noiselevel=-1)
+
del specials["LDPATH"]
notice = "# THIS FILE IS AUTOMATICALLY GENERATED BY env-update.\n"
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: a777662890a7552927ee794506a92ddffc43523e
Author: Sam James gentoo org>
AuthorDate: Tue Jul 25 03:19:14 2023 +
Commit: Sam James gentoo org>
CommitDate: Wed Jul 26 07:58:31 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7776628
lib: env_update: use more f-strings
Signed-off-by: Sam James gentoo.org>
lib/portage/util/env_update.py | 10 --
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py
index a7015669e..9d3b5bb25 100644
--- a/lib/portage/util/env_update.py
+++ b/lib/portage/util/env_update.py
@@ -361,19 +361,17 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
# an older package installed ON TOP of a newer version will cause
ldconfig
# to overwrite the symlinks we just made. -X means no links. After
'clean'
# we can safely create links.
-writemsg_level(
-_(">>> Regenerating %setc/ld.so.cache...\n") % (target_root,)
-)
+writemsg_level(_(f">>> Regenerating
{target_root}etc/ld.so.cache...\n"))
ret = subprocess.run(
-[f"{ldconfig}", "-X", "-r", f"{target_root}"], cwd="/"
+[ldconfig, "-X", "-r", target_root], cwd="/"
).returncode
elif ostype in ("FreeBSD", "DragonFly"):
writemsg_level(
-_(">>> Regenerating %svar/run/ld-elf.so.hints...\n") %
target_root
+_(f">>> Regenerating
{target_root}var/run/ld-elf.so.hints...\n")
)
ret = subprocess.run(
[
-f"{ldconfig}",
+ldconfig,
"-elf",
"-i",
"-f",
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 3bba408e214ae27bdf924ba90ad4b0919a85f3c8
Author: Sam James gentoo org>
AuthorDate: Tue Jul 25 03:16:36 2023 +
Commit: Sam James gentoo org>
CommitDate: Wed Jul 26 07:58:30 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3bba408e
lib: env_update: port to subprocess
os.system is a bit janky here, let's use subprocess so we're not injecting
multiple commands.
Thanks to mgorny for the suggestion.
Bug: https://bugs.gentoo.org/910376
Signed-off-by: Sam James gentoo.org>
lib/portage/util/env_update.py | 24 +++-
1 file changed, 15 insertions(+), 9 deletions(-)
diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py
index 5c2b2fdd5..a7015669e 100644
--- a/lib/portage/util/env_update.py
+++ b/lib/portage/util/env_update.py
@@ -6,7 +6,7 @@ __all__ = ["env_update"]
import errno
import glob
import stat
-import sys
+import subprocess
import time
import portage
@@ -364,18 +364,24 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env, writemsg_lev
writemsg_level(
_(">>> Regenerating %setc/ld.so.cache...\n") % (target_root,)
)
-ret = os.system(f"cd / ; {ldconfig} -X -r '{target_root}'")
+ret = subprocess.run(
+[f"{ldconfig}", "-X", "-r", f"{target_root}"], cwd="/"
+).returncode
elif ostype in ("FreeBSD", "DragonFly"):
writemsg_level(
_(">>> Regenerating %svar/run/ld-elf.so.hints...\n") %
target_root
)
-ret = os.system(
-(
-"cd / ; %s -elf -i "
-+ "-f '%svar/run/ld-elf.so.hints' '%setc/ld.so.conf'"
-)
-% (ldconfig, target_root, target_root)
-)
+ret = subprocess.run(
+[
+f"{ldconfig}",
+"-elf",
+"-i",
+"-f",
+f"{target_root}var/run/ld-elf.so.hints",
+f"{target_root}etc/ld.so.conf",
+],
+cwd="/",
+).returncode
ret = os.waitstatus_to_exitcode(ret)
if ret > 0:
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: f95d78c1636d081b906e613a0f26e2c318c80c5d Author: Berin Aniesh gmail com> AuthorDate: Sun Jun 18 04:53:01 2023 + Commit: Sam James gentoo org> CommitDate: Thu Jun 29 08:19:28 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f95d78c1 ExtractKernelVersion.py: Remove repeated/unreachable code Signed-off-by: Berin Aniesh gmail.com> Signed-off-by: Sam James gentoo.org> lib/portage/util/ExtractKernelVersion.py | 4 1 file changed, 4 deletions(-) diff --git a/lib/portage/util/ExtractKernelVersion.py b/lib/portage/util/ExtractKernelVersion.py index 5914dd020..6a6501b5d 100644 --- a/lib/portage/util/ExtractKernelVersion.py +++ b/lib/portage/util/ExtractKernelVersion.py @@ -31,16 +31,12 @@ def ExtractKernelVersion(base_dir): ) except OSError as details: return (None, str(details)) -except OSError as details: -return (None, str(details)) try: for i in range(4): lines.append(f.readline()) except OSError as details: return (None, str(details)) -except OSError as details: -return (None, str(details)) finally: f.close()
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/tests/process/, lib/_emerge/, ...
commit: efbc9c8d38b742a37968fc3b33d3dffdaf3be9f2
Author: Berin Aniesh gmail com>
AuthorDate: Fri Jun 9 02:00:45 2023 +
Commit: Sam James gentoo org>
CommitDate: Wed Jun 14 01:44:02 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=efbc9c8d
Remove dummy_threading
Portage's MSV is python 3.7 and dummy threading is deprecated
since python 3.7. So everything dummy_threading is removed.
Signed-off-by: Berin Aniesh gmail.com>
Closes: https://github.com/gentoo/portage/pull/1052
Signed-off-by: Sam James gentoo.org>
lib/_emerge/AsynchronousLock.py| 29
lib/_emerge/PollScheduler.py | 7 ++---
lib/portage/_emirrordist/MirrorDistTask.py | 6 +
lib/portage/debug.py | 8 ++
lib/portage/locks.py | 7 +++--
lib/portage/proxy/lazyimport.py| 8 ++
lib/portage/tests/locks/test_asynchronous_lock.py | 31 --
.../tests/process/test_PopenProcessBlockingIO.py | 14 +-
lib/portage/tests/util/futures/test_retry.py | 7 ++---
lib/portage/util/_async/PipeReaderBlockingIO.py| 8 ++
lib/portage/util/futures/_asyncio/__init__.py | 5 +---
11 files changed, 34 insertions(+), 96 deletions(-)
diff --git a/lib/_emerge/AsynchronousLock.py b/lib/_emerge/AsynchronousLock.py
index 1a69d0847..c9c8e7f3c 100644
--- a/lib/_emerge/AsynchronousLock.py
+++ b/lib/_emerge/AsynchronousLock.py
@@ -1,19 +1,10 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import fcntl
import logging
import sys
-
-try:
-import dummy_threading
-except ImportError:
-dummy_threading = None
-
-try:
-import threading
-except ImportError:
-threading = dummy_threading
+import threading
import portage
from portage import os
@@ -39,7 +30,6 @@ class AsynchronousLock(AsynchronousTask):
__slots__ = ("path",) + (
"_imp",
"_force_async",
-"_force_dummy",
"_force_process",
"_force_thread",
"_unlock_future",
@@ -61,14 +51,11 @@ class AsynchronousLock(AsynchronousTask):
return
if self._force_process or (
-not self._force_thread
-and (self._use_process_by_default or threading is dummy_threading)
+not self._force_thread and self._use_process_by_default
):
self._imp = _LockProcess(path=self.path, scheduler=self.scheduler)
else:
-self._imp = _LockThread(
-path=self.path, scheduler=self.scheduler,
_force_dummy=self._force_dummy
-)
+self._imp = _LockThread(path=self.path, scheduler=self.scheduler)
self._imp.addExitListener(self._imp_exit)
self._imp.start()
@@ -115,19 +102,13 @@ class _LockThread(AbstractPollTask):
using a background thread. After the lock is acquired, the thread
writes to a pipe in order to notify a poll loop running in the main
thread.
-
-If the threading module is unavailable then the dummy_threading
-module will be used, and the lock will be acquired synchronously
-(before the start() method returns).
"""
-__slots__ = ("path",) + ("_force_dummy", "_lock_obj", "_thread",
"_unlock_future")
+__slots__ = ("path",) + ("_lock_obj", "_thread", "_unlock_future")
def _start(self):
self._registered = True
threading_mod = threading
-if self._force_dummy:
-threading_mod = dummy_threading
self._thread = threading_mod.Thread(target=self._run_lock)
self._thread.daemon = True
self._thread.start()
diff --git a/lib/_emerge/PollScheduler.py b/lib/_emerge/PollScheduler.py
index 73f2affde..e5dffd8af 100644
--- a/lib/_emerge/PollScheduler.py
+++ b/lib/_emerge/PollScheduler.py
@@ -1,10 +1,7 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-try:
-import threading
-except ImportError:
-import dummy_threading as threading
+import threading
from portage.util.futures import asyncio
from portage.util._async.SchedulerInterface import SchedulerInterface
diff --git a/lib/portage/_emirrordist/MirrorDistTask.py
b/lib/portage/_emirrordist/MirrorDistTask.py
index 5b2543c13..2925394a8 100644
--- a/lib/portage/_emirrordist/MirrorDistTask.py
+++ b/lib/portage/_emirrordist/MirrorDistTask.py
@@ -4,11 +4,7 @@
import errno
import logging
import time
-
-try:
-import threading
-except ImportError:
-import dummy_threading as threading
+import threading
import portage
from portage import os
diff --git a/lib/portage/debug.py b/lib/portage/debug.py
index b7106b799..ee2dc13e7 100644
--- a/lib/portage/debug.py
+++ b/lib/portage/debug.py
@@ -1,13 +1,9 @@
-# Copyright
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 0911ea932d81020abe87774ff4819499c0cb891a Author: Berin Aniesh gmail com> AuthorDate: Tue Jun 6 06:16:35 2023 + Commit: Sam James gentoo org> CommitDate: Tue Jun 6 06:52:40 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0911ea93 portage.util.writemsg_level: Add return type type annotation Signed-off-by: Berin Aniesh gmail.com> Closes: https://github.com/gentoo/portage/pull/1048 Signed-off-by: Sam James gentoo.org> lib/portage/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py index f0115aea3..1f8c9e94f 100644 --- a/lib/portage/util/__init__.py +++ b/lib/portage/util/__init__.py @@ -130,7 +130,7 @@ def writemsg_stdout(mystr: str, noiselevel: int = 0) -> None: writemsg(mystr, noiselevel=noiselevel, fd=sys.stdout) -def writemsg_level(msg: str, level: int = 0, noiselevel: int = 0): +def writemsg_level(msg: str, level: int = 0, noiselevel: int = 0) -> None: """ Show a message for the given level as defined by the logging module (default is 0).
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/_emerge/
commit: 1ed0fd5f7c76f8c099a7f5978f1745e029e5e489 Author: Berin Aniesh gmail com> AuthorDate: Tue Jun 6 05:54:33 2023 + Commit: Sam James gentoo org> CommitDate: Tue Jun 6 06:52:40 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1ed0fd5f portage.util: Fix types and add add more details to docstring Signed-off-by: Berin Aniesh gmail.com> Signed-off-by: Sam James gentoo.org> lib/_emerge/main.py | 7 --- lib/portage/util/__init__.py | 17 - 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py index ec8d21b96..e471ac413 100644 --- a/lib/_emerge/main.py +++ b/lib/_emerge/main.py @@ -21,6 +21,7 @@ portage.proxy.lazyimport.lazyimport( from portage import os from portage.sync import _SUBMODULE_PATH_MAP +from typing import Optional, List options = [ "--alphabetical", @@ -1161,12 +1162,12 @@ def profile_check(trees, myaction): return os.EX_OK -def emerge_main(args=None): +def emerge_main(args: Optional[List[str]] = None): """ Entry point of emerge -Processes command line arguments and decides what the current run of emerge -should by creating `emerge_config` +Processes command line arguments (default: sys.argv[1:]) and decides +what the current run of emerge should by creating `emerge_config` """ if args is None: args = sys.argv[1:] diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py index e446a54f0..f0115aea3 100644 --- a/lib/portage/util/__init__.py +++ b/lib/portage/util/__init__.py @@ -72,7 +72,7 @@ import string import sys import traceback import glob -from typing import Optional +from typing import Optional, TextIO import portage @@ -97,8 +97,15 @@ def initialize_logger(level=logging.WARNING) -> None: logging.basicConfig(level=level, format="[%(levelname)-4s] %(message)s") -def writemsg(mystr: str, noiselevel: int = 0, fd=None) -> None: -"""Prints out warning and debug messages based on the noiselimit setting""" +def writemsg(mystr: str, noiselevel: int = 0, fd: Optional[TextIO] = None) -> None: +""" +Prints out warning and debug messages based on the noiselimit setting + +Takes three arguments +1. mystr: The message to write +2. noiselevel: The noiselevel of the message +3. fd: file descriptor - where to write the message to +""" global noiselimit if fd is None: fd = sys.stderr @@ -126,8 +133,8 @@ def writemsg_stdout(mystr: str, noiselevel: int = 0) -> None: def writemsg_level(msg: str, level: int = 0, noiselevel: int = 0): """ Show a message for the given level as defined by the logging module -(default is 0). - +(default is 0). + When level >= logging.WARNING then the message is sent to stderr, otherwise it is sent to stdout. The noiselevel is passed directly to writemsg().
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, /, lib/_emerge/
commit: 15fefd7e2e9b400a8a48d59badad632fe290aa5a
Author: Berin Aniesh gmail com>
AuthorDate: Mon Jun 5 14:06:24 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue Jun 6 06:52:40 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=15fefd7e
main.py, portage.util: Add docstrings and type annotations
Signed-off-by: Berin Aniesh gmail.com>
Signed-off-by: Sam James gentoo.org>
.gitignore | 4
lib/_emerge/main.py | 6 --
lib/portage/util/__init__.py | 25 -
3 files changed, 20 insertions(+), 15 deletions(-)
diff --git a/.gitignore b/.gitignore
index 7c6acadbe..03ae933b5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,7 @@ __pycache__/
setup.cfg
.*.swp
*.vim
+.mypy_cache
+dist
+*.egg-info/
+MANIFEST
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 850487d36..ec8d21b96 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -1163,8 +1163,10 @@ def profile_check(trees, myaction):
def emerge_main(args=None):
"""
-@param args: command arguments (default: sys.argv[1:])
-@type args: list
+Entry point of emerge
+
+Processes command line arguments and decides what the current run of emerge
+should by creating `emerge_config`
"""
if args is None:
args = sys.argv[1:]
diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py
index 31aba6702..e446a54f0 100644
--- a/lib/portage/util/__init__.py
+++ b/lib/portage/util/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2004-2020 Gentoo Authors
+# Copyright 2004-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.cache.mappings import UserDict
@@ -87,7 +87,7 @@ portage.proxy.lazyimport.lazyimport(
noiselimit = 0
-def initialize_logger(level=logging.WARNING):
+def initialize_logger(level=logging.WARNING) -> None:
"""Sets up basic logging of portage activities
Args:
level: the level to emit messages at ('info', 'debug', 'warning'
...)
@@ -97,7 +97,7 @@ def initialize_logger(level=logging.WARNING):
logging.basicConfig(level=level, format="[%(levelname)-4s] %(message)s")
-def writemsg(mystr, noiselevel=0, fd=None):
+def writemsg(mystr: str, noiselevel: int = 0, fd=None) -> None:
"""Prints out warning and debug messages based on the noiselimit setting"""
global noiselimit
if fd is None:
@@ -118,24 +118,23 @@ def writemsg(mystr, noiselevel=0, fd=None):
fd.flush()
-def writemsg_stdout(mystr, noiselevel=0):
+def writemsg_stdout(mystr: str, noiselevel: int = 0) -> None:
"""Prints messages stdout based on the noiselimit setting"""
writemsg(mystr, noiselevel=noiselevel, fd=sys.stdout)
-def writemsg_level(msg, level=0, noiselevel=0):
+def writemsg_level(msg: str, level: int = 0, noiselevel: int = 0):
"""
Show a message for the given level as defined by the logging module
-(default is 0). When level >= logging.WARNING then the message is
+(default is 0).
+
+When level >= logging.WARNING then the message is
sent to stderr, otherwise it is sent to stdout. The noiselevel is
passed directly to writemsg().
-
-@type msg: str
-@param msg: a message string, including newline if appropriate
-@type level: int
-@param level: a numeric logging level (see the logging module)
-@type noiselevel: int
-@param noiselevel: passed directly to writemsg
+Takes three parameters
+1. msg - the message to output
+2. level - the numeric logging level (see python's logging module)
+3. noiselevel - portage's logging level, passed directly to writemsg
"""
if level >= logging.WARNING:
fd = sys.stderr
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/dbapi/
commit: 5572b73744121f67c4e55040966bfe91a0e5fb6c
Author: gcarq protonmail com>
AuthorDate: Tue Mar 21 16:19:13 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue May 23 00:22:09 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5572b737
mergeme: Consider file mode and xattr for comparison
Implements dblink._needs_move(...) to take care of file equality
checks which consist of file mode, extended attributes and
file content.
Signed-off-by: gcarq protonmail.com>
Signed-off-by: Sam James gentoo.org>
lib/portage/dbapi/vartree.py | 32 +---
lib/portage/util/movefile.py | 29 +
2 files changed, 54 insertions(+), 7 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 327b72bed..317cf327a 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -3,8 +3,6 @@
__all__ = ["vardbapi", "vartree", "dblink"] + ["write_contents",
"tar_contents"]
-import filecmp
-
import portage
portage.proxy.lazyimport.lazyimport(
@@ -34,7 +32,7 @@ portage.proxy.lazyimport.lazyimport(
"portage.util.env_update:env_update",
"portage.util.install_mask:install_mask_dir,InstallMask,_raise_exc",
"portage.util.listdir:dircache,listdir",
-"portage.util.movefile:movefile",
+"portage.util.movefile:movefile,_cmpxattr",
"portage.util.path:first_existing,iter_parents",
"portage.util.writeable_check:get_ro_checker",
"portage.util._xattr:xattr",
@@ -95,6 +93,7 @@ from ._ContentsCaseSensitivityManager import
ContentsCaseSensitivityManager
import argparse
import errno
+import filecmp
import fnmatch
import functools
import gc
@@ -5803,10 +5802,7 @@ class dblink:
# same way. Unless moveme=0 (blocking directory)
if moveme:
# only replace the existing file if it differs, see #722270
-already_merged = os.path.exists(mydest)
-if already_merged and filecmp.cmp(mysrc, mydest,
shallow=False):
-zing = "==="
-else:
+if self._needs_move(mysrc, mydest, mymode, mydmode):
# Create hardlinks only for source files that already
exist
# as hardlinks (having identical st_dev and st_ino).
hardlink_key = (mystat.st_dev, mystat.st_ino)
@@ -5829,6 +5825,8 @@ class dblink:
return 1
hardlink_candidates.append(mydest)
zing = ">>>"
+else:
+zing = "==="
try:
self._merged_path(mydest, os.lstat(mydest))
@@ -6254,6 +6252,26 @@ class dblink:
finally:
self.unlockdb()
+def _needs_move(self, mysrc, mydest, mymode, mydmode):
+"""
+Checks whether the given file at |mysrc| needs to be moved to |mydest|
or if
+they are identical.
+
+Takes file mode and extended attributes into account.
+Should only be used for regular files.
+"""
+if not os.path.exists(mydest):
+return True
+
+if mymode != mydmode:
+return True
+
+excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
+if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs):
+return True
+
+return not filecmp.cmp(mysrc, mydest, shallow=False)
+
def merge(
mycat,
diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
index d46c56ade..e2f19ba92 100644
--- a/lib/portage/util/movefile.py
+++ b/lib/portage/util/movefile.py
@@ -105,6 +105,35 @@ def _copyxattr(src, dest, exclude=None):
)
+def _cmpxattr(src, dest, exclude=None):
+"""
+Compares extended attributes between |src| and |dest| and returns True
+if they are equal or xattrs are not supported, False otherwise
+"""
+try:
+src_attrs = xattr.list(src)
+dest_attrs = xattr.list(dest)
+except OSError as e:
+if e.errno != OperationNotSupported.errno:
+raise
+return True
+
+if src_attrs:
+if exclude is not None and isinstance(src_attrs[0], bytes):
+exclude = exclude.encode(_encodings["fs"])
+exclude = _get_xattr_excluder(exclude)
+
+src_attrs = {attr for attr in src_attrs if not exclude(attr)}
+dest_attrs = {attr for attr in dest_attrs if not exclude(attr)}
+if src_attrs != dest_attrs:
+return False
+
+for attr in src_attrs:
+if xattr.get(src, attr) != xattr.get(dest, attr):
+return False
+return True
+
+
def movefile(
src,
dest,
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 118df436abd49eecfeaf95df8a052861d463a896 Author: Michał Górny gentoo org> AuthorDate: Fri Dec 30 10:57:12 2022 + Commit: Sam James gentoo org> CommitDate: Sat Dec 31 13:33:04 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=118df436 Report a warning when unaccelerated Whirlpool is being used The warning serves two purposes: it informs the user that they should expect very slow hashing (i.e. Portage did not hang) and that they should inform the repository owner about the problem. Far from ideal but probably good enough given how rare Whirlpool-only ebuilds are. Signed-off-by: Michał Górny gentoo.org> Closes: https://github.com/gentoo/portage/pull/967 Signed-off-by: Sam James gentoo.org> lib/portage/util/whirlpool.py | 11 +++ 1 file changed, 11 insertions(+) diff --git a/lib/portage/util/whirlpool.py b/lib/portage/util/whirlpool.py index 8454a874a..d26780604 100644 --- a/lib/portage/util/whirlpool.py +++ b/lib/portage/util/whirlpool.py @@ -30,6 +30,10 @@ # pylint: disable=mixed-indentation +import warnings + +from portage.localization import _ + try: from ._whirlpool import Whirlpool as WhirlpoolExt except ImportError: @@ -47,6 +51,13 @@ class PyWhirlpool: hashed.""" def __init__(self, arg=b""): +warnings.warn( +_( +"The last-resort unaccelerated Whirlpool implementation is " +"being used. It is known to be absurdly slow. Please report " +"that the Whirlpool hash is deprecated to the repository owner." +) +) self.ctx = WhirlpoolStruct() self.update(arg)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
commit: 98f984875251f86d3a5c367765c7d6e249277a26
Author: Michał Górny gentoo org>
AuthorDate: Fri Dec 30 10:44:10 2022 +
Commit: Sam James gentoo org>
CommitDate: Sat Dec 31 13:33:04 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=98f98487
Test C whirlpool implementation separately from Python impl
Signed-off-by: Michał Górny gentoo.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/tests/util/test_whirlpool.py | 17 +++--
lib/portage/util/whirlpool.py| 2 ++
2 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/lib/portage/tests/util/test_whirlpool.py
b/lib/portage/tests/util/test_whirlpool.py
index d93467b21..ac156abf3 100644
--- a/lib/portage/tests/util/test_whirlpool.py
+++ b/lib/portage/tests/util/test_whirlpool.py
@@ -2,26 +2,31 @@
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
-from portage.util.whirlpool import Whirlpool
+from portage.util.whirlpool import CWhirlpool, PyWhirlpool
class WhirlpoolTestCase(TestCase):
-def testBundledWhirlpool(self):
+def testBundledWhirlpool(self, cls=PyWhirlpool):
self.assertEqual(
-Whirlpool(b"The quick brown fox jumps over the lazy
dog").hexdigest(),
+cls(b"The quick brown fox jumps over the lazy dog").hexdigest(),
"b97de512e91e3828b40d2b0fdce9ceb3c4a71f9bea8d88e75c4fa854df36725fd2b52eb6544edcacd6f8beddfea403cb55ae31f03ad62a5ef54e42ee82c3fb35",
)
self.assertEqual(
-Whirlpool(b"The quick brown fox jumps over the lazy
eog").hexdigest(),
+cls(b"The quick brown fox jumps over the lazy eog").hexdigest(),
"c27ba124205f72e6847f3e19834f925cc666d0974167af915bb462420ed40cc50900d85a1f923219d832357750492d5c143011a76988344c2635e69d06f2d38c",
)
self.assertEqual(
-Whirlpool(b"").hexdigest(),
+cls(b"").hexdigest(),
"19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3",
)
-w = Whirlpool()
+w = cls()
w.update(b"")
self.assertEqual(
w.hexdigest(),
"19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3",
)
+
+def testCWhirlpool(self):
+if not CWhirlpool.is_available:
+self.skipTest("C Whirlpool extension is not importable")
+self.testBundledWhirlpool(CWhirlpool)
diff --git a/lib/portage/util/whirlpool.py b/lib/portage/util/whirlpool.py
index 1431c18fb..7caa20f6e 100644
--- a/lib/portage/util/whirlpool.py
+++ b/lib/portage/util/whirlpool.py
@@ -83,6 +83,8 @@ class CWhirlpool:
may be provided; if present, this string will be automatically
hashed."""
+is_available = WhirlpoolExt is not None
+
def __init__(self, arg=b""):
self.obj = WhirlpoolExt()
self.dig = None
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, /, src/
commit: 45f2da735a473d8b800861f97d7be1ba7850187f
Author: Michał Górny gentoo org>
AuthorDate: Thu Dec 29 17:04:24 2022 +
Commit: Sam James gentoo org>
CommitDate: Thu Dec 29 19:34:36 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=45f2da73
portage.util: Add C implementation of Whirlpool hash
Add an extension module providing Whirlpool hash based on the reference
C implementation. This is necessary since Python hashlib no longer
provides Whirlpool by default with OpenSSL 3, and the Python
implementation used by Portage is unusably slow.
Signed-off-by: Michał Górny gentoo.org>
Closes: https://github.com/gentoo/portage/pull/966
Signed-off-by: Sam James gentoo.org>
lib/portage/util/whirlpool.py | 44 +-
setup.py |3 +
src/portage_util__whirlpool.c | 1141 +
3 files changed, 1187 insertions(+), 1 deletion(-)
diff --git a/lib/portage/util/whirlpool.py b/lib/portage/util/whirlpool.py
index e478922ef..b62191d91 100644
--- a/lib/portage/util/whirlpool.py
+++ b/lib/portage/util/whirlpool.py
@@ -27,12 +27,18 @@
# pylint: disable=mixed-indentation
+try:
+from ._whirlpool import Whirlpool as WhirlpoolExt
+except ImportError:
+WhirlpoolExt = None
+
+
# block_size = 64
digest_size = 64
digestsize = 64
-class Whirlpool:
+class PyWhirlpool:
"""Return a new Whirlpool object. An optional string argument
may be provided; if present, this string will be automatically
hashed."""
@@ -69,6 +75,42 @@ class Whirlpool:
return copy.deepcopy(self)
+class CWhirlpool:
+"""Return a new Whirlpool object. An optional string argument
+may be provided; if present, this string will be automatically
+hashed."""
+
+def __init__(self, arg=b""):
+self.obj = WhirlpoolExt()
+self.dig = None
+self.update(arg)
+
+def update(self, arg):
+if self.dig is not None:
+raise RuntimeError("Whirlpool object already finalized")
+self.obj.update(arg)
+
+def digest(self):
+if self.dig is None:
+self.dig = self.obj.digest()
+return self.dig
+
+def hexdigest(self):
+"""hexdigest()"""
+dig = self.digest()
+tempstr = ""
+for d in dig:
+xxx = "%02x" % (d,)
+tempstr = tempstr + xxx
+return tempstr
+
+
+if WhirlpoolExt is not None:
+Whirlpool = CWhirlpool
+else:
+Whirlpool = PyWhirlpool
+
+
def new(init=b""):
"""Return a new Whirlpool object. An optional string argument
may be provided; if present, this string will be automatically
diff --git a/setup.py b/setup.py
index 9f189cc6f..e6d64874b 100755
--- a/setup.py
+++ b/setup.py
@@ -86,6 +86,9 @@ x_c_helpers = {
"portage.util.libc": [
"src/portage_util_libc.c",
],
+"portage.util._whirlpool": [
+"src/portage_util__whirlpool.c",
+],
}
if platform.system() == "Linux":
diff --git a/src/portage_util__whirlpool.c b/src/portage_util__whirlpool.c
new file mode 100644
index 0..6c9421e56
--- /dev/null
+++ b/src/portage_util__whirlpool.c
@@ -0,0 +1,1141 @@
+/* Copyright 2022 Gentoo Authors
+ * Distributed under the terms of the GNU General Public License v2
+ */
+
+/**
+ * Based on the reference implementation
+ * by: Paulo S.L.M. Barreto and Vincent Rijmen.
+ *
+ * The reference implementation has been placed in public domain, with
+ * the following disclaimer:
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define PY_SSIZE_T_CLEAN
+#include
+#include
+#include
+#include
+#include
+#include
+
+typedef uint8_t u8;
+typedef uint32_t u32;
+typedef uint64_t u64;
+
+#define DIGESTBYTES 64
+#define DIGESTBITS (8*DIGESTBYTES) /* 512 */
+
+#define WBLOCKBYTES 64
+#define WBLOCKBITS (8*WBLOCKBYTES) /* 512 */
+
+#define LENGTHBYTES 32
+#define LENGTHBITS (8*LENGTHBYTES) /* 256 */
+
+typedef struct NESSIEstruct {
+ u8 bitLength[LENGTHBYTES]; /* global number of hashed bits (256-bit
counter) */
+ u8 buffer[WBLOCKBYTES];/* buffer of data to hash */
+ int bufferBits; /* current number of bits on the buffer
*/
+ int bufferPos; /* current
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, /
commit: f99ddf5659d143dd876ab05c5ff2ce425ffde672 Author: Florian Schmaus gentoo org> AuthorDate: Sun Nov 27 19:12:53 2022 + Commit: Sam James gentoo org> CommitDate: Wed Nov 30 22:28:58 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f99ddf56 Revert "env-update: skip PATH in systemd user environment definition" This reverts commit 21739ea1e0a793e207b7ae28d524e45582d9b2ca, as it was appearantly not necessary. Not sure what drove me into believing that this was needed… Thanks to Arsen for pointing this out. Closes: https://bugs.gentoo.org/883307 Reported-by: Arsen Arsenović aarsen.me> Signed-off-by: Florian Schmaus gentoo.org> Closes: https://github.com/gentoo/portage/pull/949 Signed-off-by: Sam James gentoo.org> NEWS | 3 +++ lib/portage/util/env_update.py | 5 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 6fd185fcb..57d9c92ee 100644 --- a/NEWS +++ b/NEWS @@ -46,6 +46,9 @@ Bug fixes: in different ways, but one example was a rebuild in / causing the same package to be added to ROOT, even when it had no other reason to be. +* env-update: Also generate PATH definition in systemd user environment file + /etc/environment.d/10-gentoo-env.conf + portage-3.0.39 (2022-11-20) -- diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py index bb0ebf84c..bac5b6e7a 100644 --- a/lib/portage/util/env_update.py +++ b/lib/portage/util/env_update.py @@ -417,11 +417,6 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, writemsg_lev systemd_gentoo_env.write(senvnotice) for env_key in env_keys: -# Skip PATH since this makes it impossible to use -# "systemctl --user import-environment PATH". -if env_key == "PATH": -continue - env_key_value = env[env_key] # Skip variables with the empty string
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, bin/ebuild-helpers/, bin/
commit: 800e09a4a1a630548ea8316e5c906c6f48af3105
Author: Sam James gentoo org>
AuthorDate: Tue Oct 18 00:11:24 2022 +
Commit: Sam James gentoo org>
CommitDate: Tue Oct 18 00:14:25 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=800e09a4
info: support more info compression suffixes
Bug: https://bugs.gentoo.org/757525
Signed-off-by: Sam James gentoo.org>
bin/ebuild-helpers/prepinfo | 6 +++---
bin/misc-functions.sh | 2 +-
lib/portage/util/_info_files.py | 2 +-
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/bin/ebuild-helpers/prepinfo b/bin/ebuild-helpers/prepinfo
index 9e509e8ca..4d200d0d3 100755
--- a/bin/ebuild-helpers/prepinfo
+++ b/bin/ebuild-helpers/prepinfo
@@ -1,10 +1,10 @@
#!/usr/bin/env bash
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2022 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
-if ! ___eapi_has_prefix_variables; then
+if ! ___eapi_has_prefix_variables ; then
ED=${D}
fi
@@ -32,7 +32,7 @@ find "${ED%/}/${infodir#/}" -type d -print0 | while read -r
-d $'\0' x ; do
[[ -e ${f} ]] && continue 2
done
- rm -f "${x}"/dir{,.info}{,.gz,.bz2}
+ rm -f "${x}"/dir{,.info}{,.gz,.bz2,.xz,.lz,.zst,.lzma,.lz4}
done
exit 0
diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index 170e60d1c..2fa61cf37 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -253,7 +253,7 @@ install_qa_check() {
fi
# Portage regenerates this on the installed system.
- rm -f "${ED%/}"/usr/share/info/dir{,.gz,.bz2} || die "rm failed!"
+ rm -f "${ED%/}"/usr/share/info/dir{,.gz,.bz2,.xz,.lz,.zst,.lzma,.lz4}
|| die "rm failed!"
}
__dyn_instprep() {
diff --git a/lib/portage/util/_info_files.py b/lib/portage/util/_info_files.py
index 528b273d9..4cea4f657 100644
--- a/lib/portage/util/_info_files.py
+++ b/lib/portage/util/_info_files.py
@@ -36,7 +36,7 @@ def chk_updated_info_files(root, infodirs, prev_mtimes):
if portage.util.noiselimit >= 0:
out.einfo("Regenerating GNU info directory index...")
-dir_extensions = ("", ".gz", ".bz2")
+dir_extensions = ("", ".gz", ".bz2", ".xz", ".lz", ".lz4", ".zst",
".lzma")
icount = 0
badcount = 0
errmsg = ""
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, bin/
commit: 6a47cc7bcf49b7c17fdca31969b6d6aeff84332c
Author: Sam James gentoo org>
AuthorDate: Mon Oct 17 18:37:03 2022 +
Commit: Sam James gentoo org>
CommitDate: Mon Oct 17 18:37:54 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6a47cc7b
xz: add workaround for < xz 5.3.3_alpha 32-bit issue
With older versions of xz, xz -T0 will on 32-bit systems
try to allocate too much memory and bail out.
After discussing with upstream, we set --memlimit-compress=50% as a
way to make things work with both older & newer versions. This limiting
is essentially already present with >= 5.3.3_alpha as -T0 includes it.
Note that we don't need to do this for decompression as it's automatic
there for newer versions and older versions -Tn a no-op.
Signed-off-by: Sam James gentoo.org>
bin/ecompress | 2 +-
lib/portage/util/compression_probe.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/bin/ecompress b/bin/ecompress
index 977f8d7b4..e78d6931f 100755
--- a/bin/ecompress
+++ b/bin/ecompress
@@ -128,7 +128,7 @@ if [[ ${PORTAGE_COMPRESS_FLAGS+set} != "set" ]] ; then
# See: https://bugs.gentoo.org/672916
# Setting '--rm' will remove the source files after a
successful compression.
lz4) PORTAGE_COMPRESS_FLAGS="-m --rm";;
- xz) PORTAGE_COMPRESS_FLAGS="-9 -T$(___makeopts_jobs)";;
+ xz) PORTAGE_COMPRESS_FLAGS="-9 -T$(___makeopts_jobs)
--memlimit-compress=50%";;
zstd) PORTAGE_COMPRESS_FLAGS="-q --rm -T$(___makeopts_jobs)";;
esac
fi
diff --git a/lib/portage/util/compression_probe.py
b/lib/portage/util/compression_probe.py
index bbff5a3c6..ddd8cf9ad 100644
--- a/lib/portage/util/compression_probe.py
+++ b/lib/portage/util/compression_probe.py
@@ -37,7 +37,7 @@ _compressors = {
"package": "app-arch/lzop",
},
"xz": {
-"compress": "xz -T{JOBS} ${BINPKG_COMPRESS_FLAGS}",
+"compress": "xz -T{JOBS} --memlimit-compress=50%
${BINPKG_COMPRESS_FLAGS}",
"decompress": "xz -T{JOBS} -d",
"package": "app-arch/xz-utils",
},
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
commit: e22cf80ec99c52c1192ecf99f6b66d1d892f3f86
Author: David Palao gmail com>
AuthorDate: Fri Jun 3 09:13:17 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue Jun 7 23:47:56 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e22cf80e
improvement(pylint/black): some formatting issues fixed
Signed-off-by: David Palao gmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/tests/util/test_mtimedb.py | 126 ++---
lib/portage/util/mtimedb.py| 14 ++--
2 files changed, 59 insertions(+), 81 deletions(-)
diff --git a/lib/portage/tests/util/test_mtimedb.py
b/lib/portage/tests/util/test_mtimedb.py
index 09d72049f..d80b4f1da 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -9,7 +9,6 @@ from portage.tests import TestCase
import portage
from portage.data import portage_gid, uid
from portage.util.mtimedb import MtimeDB, _MTIMEDBKEYS
-from portage.exception import DigestException
# Some data for the fixtures:
@@ -210,15 +209,12 @@ class MtimeDBTestCase(TestCase):
_TWO_RESUME_LISTS_JSON,
)
for contents in all_fixtures:
-with patch(
-"portage.util.mtimedb.open", mock_open(read_data=contents)
-):
+with patch("portage.util.mtimedb.open",
mock_open(read_data=contents)):
mtimedb = MtimeDB("/path/to/mtimedb")
self.assertLessEqual(set(mtimedb.keys()), _MTIMEDBKEYS)
def test_default_values(self):
-with patch("portage.util.mtimedb.open",
- mock_open(read_data=_EMPTY_FILE)):
+with patch("portage.util.mtimedb.open",
mock_open(read_data=_EMPTY_FILE)):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertEqual(mtimedb["starttime"], 0)
self.assertEqual(mtimedb["version"], "")
@@ -227,88 +223,75 @@ class MtimeDBTestCase(TestCase):
self.assertEqual(mtimedb["updates"], {})
def test_instances_keep_a_deepcopy_of_clean_data(self):
-with patch("portage.util.mtimedb.open",
- mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+with patch(
+"portage.util.mtimedb.open",
mock_open(read_data=_ONE_RESUME_LIST_JSON)
+):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertEqual(dict(mtimedb), dict(mtimedb._clean_data))
self.assertIsNot(mtimedb, mtimedb._clean_data)
def test_load_data_called_at_instance_creation_time(self):
-with patch("portage.util.mtimedb.open",
- mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+with patch(
+"portage.util.mtimedb.open",
mock_open(read_data=_ONE_RESUME_LIST_JSON)
+):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertEqual(
mtimedb["info"],
{
- "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.37/info": (
-1711787325),
- "/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info": (
-1735158257),
- "/usr/share/info": 1650633847
- }
+"/usr/share/binutils-data/x86_64-pc-linux-gnu/2.37/info":
(1711787325),
+"/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info":
(1735158257),
+"/usr/share/info": 1650633847,
+},
)
self.assertEqual(
mtimedb["ldpath"],
{
- "/lib": 1748456830,
- "/lib64": 1750523381,
- "/usr/lib": 1750461195,
- "/usr/lib/llvm/13/lib64": 1747003135,
- "/usr/lib/rust/lib": 1750461173,
- "/usr/lib64": 1750881821,
- "/usr/local/lib": 1711784303,
- "/usr/local/lib64": 1711784303
- }
+"/lib": 1748456830,
+"/lib64": 1750523381,
+"/usr/lib": 1750461195,
+"/usr/lib/llvm/13/lib64": 1747003135,
+"/usr/lib/rust/lib": 1750461173,
+"/usr/lib64": 1750881821,
+"/usr/local/lib": 1711784303,
+"/usr/local/lib64": 1711784303,
+},
)
self.assertEqual(
mtimedb["resume"],
{
- "favorites": [
- "@world"
- ],
- "mergelist": [
- [
- "ebuild",
- "/",
- "some-cat/some-package-1.2.3-r4",
- "merge"
- ],
- [
- "ebuild",
- "/",
- "another-cat/another-package-4.3.2-r1",
- "merge"
- ]
- ],
-
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
commit: 4540f2b0a6529705343f493abb478370c15f101f
Author: David Palao gmail com>
AuthorDate: Mon May 30 07:00:32 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue Jun 7 23:47:55 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4540f2b0
improvement(mtimedb): class has better API
New attribute ``is_readonly`` and new method ``make_readonly`` added.
Signed-off-by: David Palao gmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/tests/util/test_mtimedb.py | 28 +---
lib/portage/util/mtimedb.py| 10 ++
2 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/lib/portage/tests/util/test_mtimedb.py
b/lib/portage/tests/util/test_mtimedb.py
index a65a6be91..ecc0e7135 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -9,6 +9,8 @@ from portage.util.mtimedb import MtimeDB, _MTIMEDBKEYS
from portage.exception import DigestException
+# Some random data for the fixtures:
+
_ONE_RESUME_LIST_JSON = b"""{
"info": {
"/tmp/stage1root/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info":
1711785090,
@@ -213,7 +215,7 @@ _TWO_RESUME_LISTS_JSON = b"""{
class MtimeDBTestCase(TestCase):
text = b"Unit tests for MtimeDB"
-def test_instance_created_with_only_expected_keys(self):
+def test_instances_are_created_with_only_expected_keys(self):
all_fixtures = (
_ONE_RESUME_LIST_JSON,
_EMPTY_FILE,
@@ -227,7 +229,7 @@ class MtimeDBTestCase(TestCase):
mtimedb = MtimeDB("/path/to/mtimedb")
self.assertLessEqual(set(mtimedb.keys()), _MTIMEDBKEYS)
-def test_instance_has_default_values(self):
+def test_instances_have_default_values(self):
with patch("portage.util.mtimedb.open",
mock_open(read_data=_EMPTY_FILE)):
mtimedb = MtimeDB("/some/path/mtimedb")
@@ -237,7 +239,7 @@ class MtimeDBTestCase(TestCase):
self.assertEqual(mtimedb["ldpath"], {})
self.assertEqual(mtimedb["updates"], {})
-def test_instance_has_a_deepcopy_of_clean_data(self):
+def test_instances_keep_a_deepcopy_of_clean_data(self):
with patch("portage.util.mtimedb.open",
mock_open(read_data=_ONE_RESUME_LIST_JSON)):
mtimedb = MtimeDB("/some/path/mtimedb")
@@ -274,3 +276,23 @@ class MtimeDBTestCase(TestCase):
mtimedb = MtimeDB("/some/path/mtimedb")
mtimedb.commit()
pwrite2disk.assert_not_called()
+
+def test_is_readonly_attribute(self):
+with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+mtimedb = MtimeDB("/some/path/mtimedb")
+self.assertFalse(mtimedb.is_readonly)
+
+mtimedb.filename = None
+self.assertTrue(mtimedb.is_readonly)
+
+mtimedb.filename = "/what/ever/mtimedb"
+self.assertFalse(mtimedb.is_readonly)
+
+def test_make_readonly(self):
+with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+mtimedb = MtimeDB("/some/path/mtimedb")
+mtimedb.make_readonly()
+self.assertTrue(mtimedb.is_readonly)
+self.assertIs(mtimedb.filename, None)
diff --git a/lib/portage/util/mtimedb.py b/lib/portage/util/mtimedb.py
index a6566e3f8..95da2235f 100644
--- a/lib/portage/util/mtimedb.py
+++ b/lib/portage/util/mtimedb.py
@@ -51,6 +51,16 @@ class MtimeDB(dict):
self.filename = filename
self._load(filename)
+@property
+def is_readonly(self):
+if self.filename is None:
+return True
+else:
+return False
+
+def make_readonly(self):
+self.filename = None
+
def _load(self, filename):
f = None
content = None
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: f4fd289c7e3cbbd1a17a1510b1b126d314eea053
Author: David Palao gmail com>
AuthorDate: Tue May 31 15:52:56 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue Jun 7 23:47:56 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f4fd289c
improvement(mtimedb): using is_readonly instead "if not self.filename"
Signed-off-by: David Palao gmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/mtimedb.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/util/mtimedb.py b/lib/portage/util/mtimedb.py
index 9884746f3..497c01e05 100644
--- a/lib/portage/util/mtimedb.py
+++ b/lib/portage/util/mtimedb.py
@@ -124,7 +124,7 @@ class MtimeDB(dict):
self._clean_data = copy.deepcopy(d)
def commit(self):
-if not self.filename:
+if self.is_readonly:
return
d = {}
d.update(self)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
commit: c79c8898dc4ccd85829d00e7104b8e69ae8aa7f4
Author: David Palao gmail com>
AuthorDate: Wed May 25 15:23:20 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue Jun 7 23:47:55 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c79c8898
refactor(mtimedb): MtimeDB.commit splitted into two methods
- commit itself handles only the logic of when to write to disk
- __write_to_disk performs the actual writing
Signed-off-by: David Palao gmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/tests/util/test_mtimedb.py | 37 +++---
lib/portage/util/mtimedb.py| 41 ++
2 files changed, 56 insertions(+), 22 deletions(-)
diff --git a/lib/portage/tests/util/test_mtimedb.py
b/lib/portage/tests/util/test_mtimedb.py
index e6ddf5b80..a65a6be91 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -222,13 +222,13 @@ class MtimeDBTestCase(TestCase):
)
for contents in all_fixtures:
with patch(
-'portage.util.mtimedb.open', mock_open(read_data=contents)
+"portage.util.mtimedb.open", mock_open(read_data=contents)
):
mtimedb = MtimeDB("/path/to/mtimedb")
self.assertLessEqual(set(mtimedb.keys()), _MTIMEDBKEYS)
def test_instance_has_default_values(self):
-with patch('portage.util.mtimedb.open',
+with patch("portage.util.mtimedb.open",
mock_open(read_data=_EMPTY_FILE)):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertEqual(mtimedb["starttime"], 0)
@@ -238,8 +238,39 @@ class MtimeDBTestCase(TestCase):
self.assertEqual(mtimedb["updates"], {})
def test_instance_has_a_deepcopy_of_clean_data(self):
-with patch('portage.util.mtimedb.open',
+with patch("portage.util.mtimedb.open",
mock_open(read_data=_ONE_RESUME_LIST_JSON)):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertEqual(dict(mtimedb), dict(mtimedb._clean_data))
self.assertIsNot(mtimedb, mtimedb._clean_data)
+
+@patch("portage.util.mtimedb.MtimeDB._MtimeDB__write_to_disk")
+def test_commit_writes_to_disk_if_needed_and_possible(self, pwrite2disk):
+with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_EMPTY_FILE)):
+mtimedb = MtimeDB("/some/path/mtimedb")
+mtimedb.commit()
+pwrite2disk.assert_not_called()
+mtimedb["updates"]["/long/path/1Q-2021"] = 1739992409
+d = {}
+d.update(mtimedb)
+mtimedb.commit()
+pwrite2disk.assert_called_once_with(d)
+
+@patch("portage.util.mtimedb.MtimeDB._MtimeDB__write_to_disk")
+def test_commit_does_not_write_to_disk_if_no_file(self, pwrite2disk):
+with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_EMPTY_FILE)):
+mtimedb = MtimeDB("/some/path/mtimedb")
+mtimedb["updates"]["/long/path/1Q-2021"] = 1739992409
+mtimedb.filename = None
+mtimedb.commit()
+pwrite2disk.assert_not_called()
+
+@patch("portage.util.mtimedb.MtimeDB._MtimeDB__write_to_disk")
+def test_commit_does_not_write_to_disk_if_no_changes(self, pwrite2disk):
+with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_EMPTY_FILE)):
+mtimedb = MtimeDB("/some/path/mtimedb")
+mtimedb.commit()
+pwrite2disk.assert_not_called()
diff --git a/lib/portage/util/mtimedb.py b/lib/portage/util/mtimedb.py
index 3aab0b90b..a6566e3f8 100644
--- a/lib/portage/util/mtimedb.py
+++ b/lib/portage/util/mtimedb.py
@@ -120,24 +120,27 @@ class MtimeDB(dict):
d.update(self)
# Only commit if the internal state has changed.
if d != self._clean_data:
-d["version"] = str(portage.VERSION)
-try:
-f = atomic_ofstream(self.filename, mode="wb")
-except EnvironmentError:
-pass
-else:
-if self._json_write:
-f.write(
-_unicode_encode(
-json.dumps(d, **self._json_write_opts),
-encoding=_encodings["repo.content"],
-errors="strict",
-)
+self.__write_to_disk(d)
+
+def __write_to_disk(self, d):
+d["version"] = str(portage.VERSION)
+try:
+f = atomic_ofstream(self.filename, mode="wb")
+except EnvironmentError:
+pass
+else:
+if self._json_write:
+f.write(
+_unicode_encode(
+json.dumps(d, **self._json_write_opts),
+encoding=_encodings["repo.content"],
+errors="strict",
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
commit: 8a0a42ceb530b277e17220614f8c34bc8d0e503c
Author: David Palao gmail com>
AuthorDate: Tue May 31 15:30:48 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue Jun 7 23:47:55 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8a0a42ce
test(MtimeDB): added some unit tests for __write_to_disk and for _load
Signed-off-by: David Palao gmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/tests/util/test_mtimedb.py | 122 +++--
lib/portage/util/mtimedb.py| 1 +
2 files changed, 102 insertions(+), 21 deletions(-)
diff --git a/lib/portage/tests/util/test_mtimedb.py
b/lib/portage/tests/util/test_mtimedb.py
index ecc0e7135..09d72049f 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -2,31 +2,28 @@
# Distributed under the terms of the GNU General Public License v2
from unittest.mock import patch, mock_open
+import json
from portage.tests import TestCase
+import portage
+from portage.data import portage_gid, uid
from portage.util.mtimedb import MtimeDB, _MTIMEDBKEYS
from portage.exception import DigestException
-# Some random data for the fixtures:
+# Some data for the fixtures:
_ONE_RESUME_LIST_JSON = b"""{
"info": {
-
"/tmp/stage1root/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info":
1711785090,
-
"/tmp/stage1root/usr/share/gcc-data/x86_64-pc-linux-gnu/9.3.0/info": 1711785090,
- "/tmp/stage1root/usr/share/info": 1611785090,
- "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info":
1711787325,
+ "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.37/info":
1711787325,
"/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info":
1735158257,
- "/usr/share/gcc-data/x86_64-pc-linux-gnu/9.3.0/info":
1711787325,
"/usr/share/info": 1650633847
},
"ldpath": {
"/lib": 1748456830,
"/lib64": 1750523381,
"/usr/lib": 1750461195,
- "/usr/lib/llvm/11/lib64": 1723048948,
- "/usr/lib/llvm/12/lib64": 1730499781,
"/usr/lib/llvm/13/lib64": 1747003135,
"/usr/lib/rust/lib": 1750461173,
"/usr/lib64": 1750881821,
@@ -66,12 +63,8 @@ _ONE_RESUME_LIST_JSON = b"""{
},
"starttime": 0,
"updates": {
- "/var/db/repos/gentoo/profiles/updates/1Q-2021": 1739992409,
"/var/db/repos/gentoo/profiles/updates/1Q-2022": 1747854791,
- "/var/db/repos/gentoo/profiles/updates/2Q-2021": 1724404379,
"/var/db/repos/gentoo/profiles/updates/2Q-2022": 1752846209,
- "/var/db/repos/gentoo/profiles/updates/3Q-2021": 1741119203,
- "/var/db/repos/gentoo/profiles/updates/4Q-2020": 1709167362,
"/var/db/repos/gentoo/profiles/updates/4Q-2021": 1742787797
},
"version": "3.0.30"
@@ -85,8 +78,6 @@ _PARTIAL_FILE_JSON = b"""{
"/lib": 1748456830,
"/lib64": 1750523381,
"/usr/lib": 1750461195,
- "/usr/lib/llvm/11/lib64": 1723048948,
- "/usr/lib/llvm/12/lib64": 1730499781,
"/usr/lib/llvm/13/lib64": 1747003135,
"/usr/lib/rust/lib": 1750461173,
"/usr/lib64": 1750881821,
@@ -130,7 +121,7 @@ _PARTIAL_FILE_JSON = b"""{
_TWO_RESUME_LISTS_JSON = b"""{
"info": {
- "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info":
1711787325,
+ "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.37/info":
1711787325,
"/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info":
1735158257,
"/usr/share/info": 1650633847
},
@@ -199,12 +190,8 @@ _TWO_RESUME_LISTS_JSON = b"""{
},
"starttime": 0,
"updates": {
- "/var/db/repos/gentoo/profiles/updates/1Q-2021": 1739992409,
"/var/db/repos/gentoo/profiles/updates/1Q-2022": 1747854791,
- "/var/db/repos/gentoo/profiles/updates/2Q-2021": 1724404379,
"/var/db/repos/gentoo/profiles/updates/2Q-2022": 1752846209,
- "/var/db/repos/gentoo/profiles/updates/3Q-2021": 1741119203,
- "/var/db/repos/gentoo/profiles/updates/4Q-2020": 1709167362,
"/var/db/repos/gentoo/profiles/updates/4Q-2021": 1742787797
},
"version": "3.0.30"
@@ -215,7 +202,7 @@ _TWO_RESUME_LISTS_JSON = b"""{
class MtimeDBTestCase(TestCase):
text = b"Unit tests for MtimeDB"
-def test_instances_are_created_with_only_expected_keys(self):
+def test_instances_created_with_only_expected_keys(self):
all_fixtures = (
_ONE_RESUME_LIST_JSON,
_EMPTY_FILE,
@@ -229,7 +216,7 @@ class MtimeDBTestCase(TestCase):
mtimedb =
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 92c63ef74039e2361a4912ead5416b08cdfa8ac8
Author: David Palao gmail com>
AuthorDate: Sun May 22 09:40:45 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue Jun 7 23:47:54 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=92c63ef7
refactor(mtimedb): some little changes:
- interpolated strings (%...) replaced by f-strings
- some little simplifications
- factor out some constants to prepare for unittesting
Signed-off-by: David Palao gmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/mtimedb.py | 44 ++--
1 file changed, 22 insertions(+), 22 deletions(-)
diff --git a/lib/portage/util/mtimedb.py b/lib/portage/util/mtimedb.py
index 7a7fe6784..df9bd494c 100644
--- a/lib/portage/util/mtimedb.py
+++ b/lib/portage/util/mtimedb.py
@@ -23,7 +23,23 @@ from portage.localization import _
from portage.util import apply_secpass_permissions, atomic_ofstream, writemsg
+_MTIMEDBKEYS = {
+"info",
+"ldpath",
+"resume",
+"resume_backup",
+"starttime",
+"updates",
+"version",
+}
+
+
class MtimeDB(dict):
+"""The MtimeDB class is used to interact with a file storing the
+current resume lists.
+It is a subclass of ``dict`` and it reads from/writes to JSON, by
+default, althouth it can be configured to use ``pickle``.
+"""
# JSON read support has been available since portage-2.1.10.49.
_json_write = True
@@ -46,13 +62,13 @@ class MtimeDB(dict):
pass
else:
writemsg(
-_("!!! Error loading '%s': %s\n") % (filename, e),
noiselevel=-1
+_(f"!!! Error loading '{filename}': {e}\n"), noiselevel=-1
)
finally:
if f is not None:
f.close()
-d = None
+d = {}
if content:
try:
d = json.loads(
@@ -75,12 +91,9 @@ class MtimeDB(dict):
raise
except Exception:
writemsg(
-_("!!! Error loading '%s': %s\n") % (filename, e),
noiselevel=-1
+_(f"!!! Error loading '{filename}': {e}\n"),
noiselevel=-1
)
-if d is None:
-d = {}
-
if "old" in d:
d["updates"] = d["old"]
del d["old"]
@@ -92,22 +105,9 @@ class MtimeDB(dict):
for k in ("info", "ldpath", "updates"):
d.setdefault(k, {})
-mtimedbkeys = set(
-(
-"info",
-"ldpath",
-"resume",
-"resume_backup",
-"starttime",
-"updates",
-"version",
-)
-)
-
-for k in list(d):
-if k not in mtimedbkeys:
-writemsg(_("Deleting invalid mtimedb key: %s\n") % str(k))
-del d[k]
+for k in (d.keys()-_MTIMEDBKEYS):
+writemsg(_(f"Deleting invalid mtimedb key: {k}\n"))
+del d[k]
self.update(d)
self._clean_data = copy.deepcopy(d)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 1f33f97b9e9c132d77d586d10bfe6ba0d3123050
Author: Thomas Bracht Laumann Jespersen laumann xyz>
AuthorDate: Thu May 19 09:50:05 2022 +
Commit: Michał Górny gentoo org>
CommitDate: Sat May 21 10:11:50 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1f33f97b
lib/portage/util: fix bundled whirlpool on empty bytestring input
The WhirlpoolAdd function did not consider zero-length input, so calls
to update(b'') would produce out-of-bounds errors. This was not covered
by any tests, because the constructor implicitly skipped the call to
update on zero-length input.
Add check for zero-length input to WhirlpoolAdd, and have the Whirlpool
constructor skip calling update() only if arg is None.
Closes: https://bugs.gentoo.org/846389
Signed-off-by: Thomas Bracht Laumann Jespersen laumann.xyz>
Closes: https://github.com/gentoo/portage/pull/832
Signed-off-by: Michał Górny gentoo.org>
lib/portage/util/whirlpool.py | 16 +++-
1 file changed, 11 insertions(+), 5 deletions(-)
diff --git a/lib/portage/util/whirlpool.py b/lib/portage/util/whirlpool.py
index de344d8eb..9178d70c7 100644
--- a/lib/portage/util/whirlpool.py
+++ b/lib/portage/util/whirlpool.py
@@ -37,11 +37,9 @@ class Whirlpool:
may be provided; if present, this string will be automatically
hashed."""
-def __init__(self, arg=None):
+def __init__(self, arg=b""):
self.ctx = WhirlpoolStruct()
-if arg:
-self.update(arg)
-self.digest_status = 0
+self.update(arg)
def update(self, arg):
"""update(arg)"""
@@ -71,7 +69,7 @@ class Whirlpool:
return copy.deepcopy(self)
-def new(init=None):
+def new(init=b""):
"""Return a new Whirlpool object. An optional string argument
may be provided; if present, this string will be automatically
hashed."""
@@ -2183,6 +2181,8 @@ def WhirlpoolInit(ctx):
def WhirlpoolAdd(source, sourceBits, ctx):
if not isinstance(source, bytes):
raise TypeError("Expected %s, got %s" % (bytes, type(source)))
+if sourceBits == 0:
+return
carry = 0
value = sourceBits
@@ -2350,3 +2350,9 @@ if __name__ == "__main__":
Whirlpool(b"").hexdigest()
==
"19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3"
)
+w = Whirlpool()
+w.update(b"")
+assert (
+w.hexdigest()
+==
"19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3"
+)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: fa82725ba03ffa5edf0b82db7307c87fc97e83f2 Author: Mike Frysinger chromium org> AuthorDate: Wed Mar 30 06:32:07 2022 + Commit: Sam James gentoo org> CommitDate: Fri Apr 15 02:46:19 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa82725b movefile: merge symlinks atomically Since POSIX allows renaming symlinks with symlinks, use that to get atomic updates. This is faster and less flaky to parallel processes: removing a live symlink and then recreating it leaves a small window where other things might try to use it but fail. [sam: cherry-picked from chromiumos' third_party/portage_tool repo] (cherry picked from commit cherry-pick 80ed2dac95db81acac8043e6685d0a853a08d268) Bug: https://bugs.gentoo.org/836400 Signed-off-by: Sam James gentoo.org> Closes: https://github.com/gentoo/portage/pull/816 Signed-off-by: Sam James gentoo.org> lib/portage/util/movefile.py | 16 +--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py index ddafe5571..4dc08af26 100644 --- a/lib/portage/util/movefile.py +++ b/lib/portage/util/movefile.py @@ -171,7 +171,7 @@ def movefile( bsd_chflags.chflags(os.path.dirname(dest), 0) if destexists: -if stat.S_ISLNK(dstat[stat.ST_MODE]): +if not stat.S_ISLNK(sstat[stat.ST_MODE]) and stat.S_ISLNK(dstat[stat.ST_MODE]): try: os.unlink(dest) destexists = 0 @@ -185,8 +185,18 @@ def movefile( target = os.readlink(src) if mysettings and "D" in mysettings and target.startswith(mysettings["D"]): target = target[len(mysettings["D"]) - 1 :] -if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): -os.unlink(dest) +# Atomically update the path if it exists. +try: +os.rename(src, dest) +return sstat.st_mtime_ns +except OSError: +# If it failed due to cross-link device, fallthru below. +# Clear the target first so we can create it. +try: +os.unlink(dest) +except FileNotFoundError: +pass + try: if selinux_enabled: selinux.symlink(target, dest, src)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, bin/, lib/portage/tests/
commit: 781aefd8be33c3bd699e5a665225013c5f76b531 Author: John Helmert III gentoo org> AuthorDate: Wed Mar 9 19:48:11 2022 + Commit: Mike Gilbert gentoo org> CommitDate: Wed Mar 9 19:57:34 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=781aefd8 Drop -b from shebangs The switch to using '/usr/bin/env python' shebangs doesn't naively work with arguments after 'python', so the shebangs were broken. '-b' is of dubious necessity anyway, so let's just drop it. Fixes: 41f4f6d25019c4f944b8d9576d96482f92231fda Signed-off-by: John Helmert III gentoo.org> Signed-off-by: Mike Gilbert gentoo.org> bin/archive-conf| 2 +- bin/binhost-snapshot| 2 +- bin/check-implicit-pointer-usage.py | 2 +- bin/chmod-lite.py | 2 +- bin/chpathtool.py | 2 +- bin/clean_locks | 2 +- bin/dispatch-conf | 2 +- bin/dohtml.py | 2 +- bin/doins.py| 2 +- bin/ebuild | 2 +- bin/ebuild-ipc.py | 2 +- bin/egencache | 2 +- bin/emaint | 2 +- bin/emerge | 2 +- bin/emirrordist | 2 +- bin/env-update | 2 +- bin/filter-bash-environment.py | 2 +- bin/fixpackages | 2 +- bin/glsa-check | 2 +- bin/gpkg-helper.py | 2 +- bin/install.py | 2 +- bin/lock-helper.py | 2 +- bin/portageq| 2 +- bin/quickpkg| 2 +- bin/regenworld | 2 +- bin/shelve-utils| 2 +- bin/xattr-helper.py | 2 +- bin/xpak-helper.py | 2 +- lib/portage/tests/runTests.py | 2 +- lib/portage/util/changelog.py | 2 +- 30 files changed, 30 insertions(+), 30 deletions(-) diff --git a/bin/archive-conf b/bin/archive-conf index beb4fcd23..3f7d186fe 100755 --- a/bin/archive-conf +++ b/bin/archive-conf @@ -1,4 +1,4 @@ -#!/usr/bin/env python -b +#!/usr/bin/env python # Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 diff --git a/bin/binhost-snapshot b/bin/binhost-snapshot index 3461a301c..4022cb32c 100755 --- a/bin/binhost-snapshot +++ b/bin/binhost-snapshot @@ -1,4 +1,4 @@ -#!/usr/bin/env python -b +#!/usr/bin/env python # Copyright 2010-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 diff --git a/bin/check-implicit-pointer-usage.py b/bin/check-implicit-pointer-usage.py index 61eecd65b..06b666c88 100755 --- a/bin/check-implicit-pointer-usage.py +++ b/bin/check-implicit-pointer-usage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python -b +#!/usr/bin/env python # Ripped from HP and updated from Debian # Update by Gentoo to support unicode output diff --git a/bin/chmod-lite.py b/bin/chmod-lite.py index 602b3600b..517a55bd9 100755 --- a/bin/chmod-lite.py +++ b/bin/chmod-lite.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python -b +#!/usr/bin/env python # Copyright 2015 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 diff --git a/bin/chpathtool.py b/bin/chpathtool.py index 2d4993716..d658e5012 100755 --- a/bin/chpathtool.py +++ b/bin/chpathtool.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python -b +#!/usr/bin/env python # Copyright 2011-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 diff --git a/bin/clean_locks b/bin/clean_locks index 88e1b3325..b80213911 100755 --- a/bin/clean_locks +++ b/bin/clean_locks @@ -1,4 +1,4 @@ -#!/usr/bin/env python -b +#!/usr/bin/env python # Copyright 1999-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 diff --git a/bin/dispatch-conf b/bin/dispatch-conf index c0e7ffc1e..9490197d3 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -1,4 +1,4 @@ -#!/usr/bin/env python -b +#!/usr/bin/env python # Copyright 1999-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 diff --git a/bin/dohtml.py b/bin/dohtml.py index e7cfa591e..86cda72aa 100755 --- a/bin/dohtml.py +++ b/bin/dohtml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python -b +#!/usr/bin/env python # Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 diff --git a/bin/doins.py b/bin/doins.py index 42ea25078..087756218 100644 --- a/bin/doins.py +++ b/bin/doins.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python -b +#!/usr/bin/env python # Copyright 2017-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 # diff --git a/bin/ebuild b/bin/ebuild index bdedab9cb..bab8d52ad 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -1,4 +1,4 @@ -#!/usr/bin/env python -b +#!/usr/bin/env python # Copyright
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/, lib/portage/util/, lib/portage/
commit: a5092f11f31b6a9d54a34423088a7bb8e70bc1c6
Author: Michał Górny gentoo org>
AuthorDate: Wed Dec 8 21:16:19 2021 +
Commit: Michał Górny gentoo org>
CommitDate: Wed Dec 8 21:16:19 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a5092f11
Reformat with black 21.12b0
Signed-off-by: Michał Górny gentoo.org>
lib/portage/process.py | 2 --
lib/portage/util/_xattr.py | 1 -
lib/portage/util/futures/unix_events.py | 1 -
3 files changed, 4 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index e1bd2314e..84e09f8ec 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -84,7 +84,6 @@ if _fd_dir is not None:
raise
return range(max_fd_limit)
-
elif os.path.isdir("/proc/%s/fd" % portage.getpid()):
# In order for this function to work in forked subprocesses,
# os.getpid() must be called from inside the function.
@@ -95,7 +94,6 @@ elif os.path.isdir("/proc/%s/fd" % portage.getpid()):
if fd.isdigit()
)
-
else:
def get_open_fds():
diff --git a/lib/portage/util/_xattr.py b/lib/portage/util/_xattr.py
index ff0a7d8c5..f4d3ad733 100644
--- a/lib/portage/util/_xattr.py
+++ b/lib/portage/util/_xattr.py
@@ -169,7 +169,6 @@ if hasattr(os, "getxattr"):
def list(item, nofollow=False, namespace=None):
return os.listxattr(item, follow_symlinks=not nofollow)
-
else:
try:
# Maybe we have the xattr module.
diff --git a/lib/portage/util/futures/unix_events.py
b/lib/portage/util/futures/unix_events.py
index f2f01f0c6..c9855aefb 100644
--- a/lib/portage/util/futures/unix_events.py
+++ b/lib/portage/util/futures/unix_events.py
@@ -23,7 +23,6 @@ if hasattr(os, "set_blocking"):
def _set_nonblocking(fd):
os.set_blocking(fd, False)
-
else:
def _set_nonblocking(fd):
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_dyn_libs/
commit: 3b3503bb2769b390e22a4fea42fa56fe4db4ddec
Author: Mike Gilbert gentoo org>
AuthorDate: Tue Nov 23 22:39:24 2021 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue Nov 23 23:22:32 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3b3503bb
portage/util/_dyn_libs/dyn_libs.py: fix symlink recursion issue
Python's glob.glob() function follows symlinks recursively. Use
os.walk() instead.
Also re-order the operands to short-circuit the directory walk when
PROVIDES is not empty.
Bug: https://bugs.gentoo.org/826982
Fixes: cba2156dba89a22f2858238013469b4d80208854
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/_dyn_libs/dyn_libs.py | 13 ++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/lib/portage/util/_dyn_libs/dyn_libs.py
b/lib/portage/util/_dyn_libs/dyn_libs.py
index 34b4bc926..ee28e8839 100644
--- a/lib/portage/util/_dyn_libs/dyn_libs.py
+++ b/lib/portage/util/_dyn_libs/dyn_libs.py
@@ -1,7 +1,15 @@
# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import glob
+import os
+
+
+def installed_dynlibs(directory):
+for _dirpath, _dirnames, filenames in os.walk(directory):
+for filename in filenames:
+if filename.endswith(".so"):
+return True
+return False
def check_dyn_libs_inconsistent(directory, provides):
@@ -17,5 +25,4 @@ def check_dyn_libs_inconsistent(directory, provides):
# but this doesn't gain us anything. We're interested in failure
# to properly parse the installed files at all, which should really
# be a global problem (e.g. bug #811462)
-installed_dynlibs = glob.glob(directory + "/**/*.so", recursive=True)
-return installed_dynlibs and not provides
+return not provides and installed_dynlibs(directory)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_dyn_libs/
commit: dcf6de8aa115cfe1ca5474b3c98efa8f75f8f47b Author: Sam James gentoo org> AuthorDate: Thu Oct 28 04:59:20 2021 + Commit: Sam James gentoo org> CommitDate: Thu Oct 28 05:00:10 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dcf6de8a lib/portage/util/_dyn_libs/dyn_libs.py: drop unused import Signed-off-by: Sam James gentoo.org> lib/portage/util/_dyn_libs/dyn_libs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/portage/util/_dyn_libs/dyn_libs.py b/lib/portage/util/_dyn_libs/dyn_libs.py index 2a014812c..34b4bc926 100644 --- a/lib/portage/util/_dyn_libs/dyn_libs.py +++ b/lib/portage/util/_dyn_libs/dyn_libs.py @@ -3,8 +3,6 @@ import glob -from portage.output import colorize - def check_dyn_libs_inconsistent(directory, provides): """Checks directory for whether any dynamic libraries were installed and
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_dyn_libs/
commit: cba2156dba89a22f2858238013469b4d80208854 Author: Sam James gentoo org> AuthorDate: Thu Oct 28 04:40:09 2021 + Commit: Sam James gentoo org> CommitDate: Thu Oct 28 04:52:18 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cba2156d portage/util/_dyn_libs/dyn_libs.py: add helper Needed for binpkg/image verification commits coming... Bug: https://bugs.gentoo.org/811462 Signed-off-by: Sam James gentoo.org> lib/portage/util/_dyn_libs/dyn_libs.py | 23 +++ 1 file changed, 23 insertions(+) diff --git a/lib/portage/util/_dyn_libs/dyn_libs.py b/lib/portage/util/_dyn_libs/dyn_libs.py new file mode 100644 index 0..2a014812c --- /dev/null +++ b/lib/portage/util/_dyn_libs/dyn_libs.py @@ -0,0 +1,23 @@ +# Copyright 2021 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import glob + +from portage.output import colorize + + +def check_dyn_libs_inconsistent(directory, provides): +"""Checks directory for whether any dynamic libraries were installed and +if PROVIDES corresponds.""" + +# Let's check if we've got inconsistent results. +# If we're installing dynamic libraries (.so files), we should +# really have a PROVIDES. +# (This is a complementary check at the point of ingestion for the +# creation check in doebuild.py) +# Note: we could check a non-empty PROVIDES against the list of .sos, +# but this doesn't gain us anything. We're interested in failure +# to properly parse the installed files at all, which should really +# be a global problem (e.g. bug #811462) +installed_dynlibs = glob.glob(directory + "/**/*.so", recursive=True) +return installed_dynlibs and not provides
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 3f2ba95a960ed66dde1a0e4cea99f5d54b018d0f Author: Zac Medico gentoo org> AuthorDate: Tue Sep 21 04:49:23 2021 + Commit: Zac Medico gentoo org> CommitDate: Tue Sep 21 04:49:45 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3f2ba95a ForkProcess: convert compat coroutine to async Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_async/ForkProcess.py | 10 -- 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/portage/util/_async/ForkProcess.py b/lib/portage/util/_async/ForkProcess.py index 674336935..e70238705 100644 --- a/lib/portage/util/_async/ForkProcess.py +++ b/lib/portage/util/_async/ForkProcess.py @@ -1,4 +1,4 @@ -# Copyright 2012-2020 Gentoo Authors +# Copyright 2012-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import fcntl @@ -10,7 +10,6 @@ import sys import portage from portage import os from portage.util.futures import asyncio -from portage.util.futures.compat_coroutine import coroutine from _emerge.SpawnProcess import SpawnProcess @@ -73,15 +72,14 @@ class ForkProcess(SpawnProcess): if self._proc_join_task is None: super(ForkProcess, self)._async_waitpid() -@coroutine -def _proc_join(self, proc, loop=None): +async def _proc_join(self, proc, loop=None): sentinel_reader = self.scheduler.create_future() self.scheduler.add_reader( proc.sentinel, lambda: sentinel_reader.done() or sentinel_reader.set_result(None), ) try: -yield sentinel_reader +await sentinel_reader finally: # If multiprocessing.Process supports the close method, then # access to proc.sentinel will raise ValueError if the @@ -101,7 +99,7 @@ class ForkProcess(SpawnProcess): proc.join(0) if proc.exitcode is not None: break -yield asyncio.sleep(self._proc_join_interval, loop=loop) +await asyncio.sleep(self._proc_join_interval, loop=loop) def _proc_join_done(self, proc, future): future.cancelled() or future.result()
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/
commit: 2f474767eaf3b042bea31bc524281e0841cb271b Author: Zac Medico gentoo org> AuthorDate: Mon Sep 20 06:44:36 2021 + Commit: Zac Medico gentoo org> CommitDate: Mon Sep 20 07:07:22 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2f474767 ExtendedFuture: Fix python3.10 DeprecationWarning DeprecationWarning: There is no current event loop Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/extendedfutures.py | 24 +--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/portage/util/futures/extendedfutures.py b/lib/portage/util/futures/extendedfutures.py index 6aa3ebbe8..c23feafb5 100644 --- a/lib/portage/util/futures/extendedfutures.py +++ b/lib/portage/util/futures/extendedfutures.py @@ -10,7 +10,13 @@ __all__ = ( "InvalidStateError", ) -from portage.util.futures.futures import Future, InvalidStateError, CancelledError +import concurrent.futures +from concurrent.futures import Future, CancelledError + +try: +from concurrent.futures import InvalidStateError +except ImportError: +from portage.util.futures.futures import InvalidStateError # Create our one time settable unset constant UNSET_CONST = Future() @@ -62,9 +68,21 @@ class ExtendedFuture(Future): default = self.default_result if default is not UNSET_CONST.result(): try: -data = super(ExtendedFuture, self).result() +data = self.result() except InvalidStateError: data = default else: -data = super(ExtendedFuture, self).result() +data = self.result() return data + +def exception(self): +try: +return super(ExtendedFuture, self).exception(timeout=0) +except concurrent.futures.TimeoutError: +raise InvalidStateError + +def result(self): +try: +return super(ExtendedFuture, self).result(timeout=0) +except concurrent.futures.TimeoutError: +raise InvalidStateError
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: 631bedffe29124d693de3b539fc908d9feec1420 Author: Zac Medico gentoo org> AuthorDate: Mon Sep 20 05:05:38 2021 + Commit: Zac Medico gentoo org> CommitDate: Mon Sep 20 05:08:47 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=631bedff _safe_loop: fix python3.10 DeprecationWarning DeprecationWarning: There is no current event loop Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/_asyncio/__init__.py | 10 +++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index c1229528a..ccf800c66 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -280,10 +280,14 @@ def _safe_loop(): loop = _thread_weakrefs.loops[thread_key] except KeyError: try: -_real_asyncio.get_event_loop() +try: +_loop = _real_asyncio.get_running_loop() +except AttributeError: +_loop = _real_asyncio.get_event_loop() except RuntimeError: -_real_asyncio.set_event_loop(_real_asyncio.new_event_loop()) -loop = _thread_weakrefs.loops[thread_key] = _AsyncioEventLoop() +_loop = _real_asyncio.new_event_loop() +_real_asyncio.set_event_loop(_loop) +loop = _thread_weakrefs.loops[thread_key] = _AsyncioEventLoop(loop=_loop) if ( _thread_weakrefs.mainloop is None
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_dyn_libs/, lib/_emerge/
commit: 1ae727e997e2c7915a4384a27c9362e59d8dd074
Author: Michał Górny gentoo org>
AuthorDate: Tue Aug 31 12:05:41 2021 +
Commit: Michał Górny gentoo org>
CommitDate: Fri Sep 3 11:26:28 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1ae727e9
post_emerge: Display all preserved libs with --verbose
Signed-off-by: Michał Górny gentoo.org>
Closes: https://github.com/gentoo/portage/pull/742
Signed-off-by: Michał Górny gentoo.org>
lib/_emerge/post_emerge.py | 2 +-
lib/portage/util/_dyn_libs/display_preserved_libs.py | 11 +++
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/lib/_emerge/post_emerge.py b/lib/_emerge/post_emerge.py
index 0c4df0d32..c540308d3 100644
--- a/lib/_emerge/post_emerge.py
+++ b/lib/_emerge/post_emerge.py
@@ -140,7 +140,7 @@ def post_emerge(myaction, myopts, myfiles,
else:
print()
print(colorize("WARN", "!!!") + " existing preserved
libs:")
- display_preserved_libs(vardbapi)
+ display_preserved_libs(vardbapi, verbose="--verbose" in
myopts)
print("Use " + colorize("GOOD", "emerge
@preserved-rebuild") +
" to rebuild packages using these libraries")
diff --git a/lib/portage/util/_dyn_libs/display_preserved_libs.py
b/lib/portage/util/_dyn_libs/display_preserved_libs.py
index 8deafc25e..5818501bb 100644
--- a/lib/portage/util/_dyn_libs/display_preserved_libs.py
+++ b/lib/portage/util/_dyn_libs/display_preserved_libs.py
@@ -6,7 +6,7 @@ import portage
from portage.output import colorize
-def display_preserved_libs(vardb):
+def display_preserved_libs(vardb, verbose=False):
MAX_DISPLAY = 3
@@ -36,7 +36,8 @@ def display_preserved_libs(vardb):
consumers.append(c)
consumers.sort()
consumer_map[f] = consumers
-
search_for_owners.update(consumers[:MAX_DISPLAY+1])
+ max_search = None if verbose else MAX_DISPLAY +
1
+ search_for_owners.update(consumers[:max_search])
owners = {}
for f in search_for_owners:
@@ -75,7 +76,9 @@ def display_preserved_libs(vardb):
# they don't need to be rebuilt (see bug
#461908).
consumers = consumers_non_preserved
- if len(consumers) == MAX_DISPLAY + 1:
+ if verbose:
+ max_display = None
+ elif len(consumers) == MAX_DISPLAY + 1:
# Display 1 extra consumer, instead of
displaying
# "used by 1 other files".
max_display = MAX_DISPLAY + 1
@@ -91,6 +94,6 @@ def display_preserved_libs(vardb):
owners_desc = ", ".join(x.mycpv for x
in owners.get(c, []))
print(colorize("WARN", " * ") + " used by
%s (%s)" % \
(c, owners_desc))
- if len(consumers) > max_display:
+ if not verbose and len(consumers) > max_display:
print(colorize("WARN", " * ") + " used by
%d other files" %
(len(consumers) - max_display))
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/, lib/portage/sync/
commit: a48532754eae8ea3dfa491968000313872b18241
Author: Florian Schmaus geekplace eu>
AuthorDate: Tue Aug 18 19:48:34 2020 +
Commit: Michał Górny gentoo org>
CommitDate: Sun Aug 22 15:32:06 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a4853275
util: add portage.util.hooks.get_hooks_from_dir()
Signed-off-by: Florian Schmaus geekplace.eu>
Signed-off-by: Michał Górny gentoo.org>
lib/portage/dispatch_conf.py | 1 +
lib/portage/sync/controller.py | 16 ++--
lib/portage/util/hooks.py | 31 +++
3 files changed, 34 insertions(+), 14 deletions(-)
diff --git a/lib/portage/dispatch_conf.py b/lib/portage/dispatch_conf.py
index 0356ba7bc..3ef659852 100644
--- a/lib/portage/dispatch_conf.py
+++ b/lib/portage/dispatch_conf.py
@@ -18,6 +18,7 @@ from portage import _encodings, os, shutil
from portage.env.loaders import KeyValuePairFileLoader
from portage.localization import _
from portage.util import shlex_split, varexpand
+from portage.util.hooks import perform_hooks
from portage.util.path import iter_parents
RCS_BRANCH = '1.1.1'
diff --git a/lib/portage/sync/controller.py b/lib/portage/sync/controller.py
index 0f42b1da9..f1d706d7e 100644
--- a/lib/portage/sync/controller.py
+++ b/lib/portage/sync/controller.py
@@ -7,8 +7,6 @@ import grp
import pwd
import warnings
-from collections import OrderedDict
-
import portage
from portage import os
from portage.progress import ProgressBar
@@ -20,9 +18,9 @@ bad = create_color_func("BAD")
warn = create_color_func("WARN")
from portage.package.ebuild.doebuild import _check_temp_dir
from portage.metadata import action_metadata
+from portage.util.hooks import get_hooks_from_dir
from portage.util._async.AsyncFunction import AsyncFunction
from portage import _unicode_decode
-from portage import util
from _emerge.CompositeTask import CompositeTask
@@ -93,17 +91,7 @@ class SyncManager:
self.module_names = self.module_controller.module_names
self.hooks = {}
for _dir in ["repo.postsync.d", "postsync.d"]:
- postsync_dir =
os.path.join(self.settings["PORTAGE_CONFIGROOT"],
- portage.USER_CONFIG_PATH, _dir)
- hooks = OrderedDict()
- for filepath in util._recursive_file_list(postsync_dir):
- name =
filepath.split(postsync_dir)[1].lstrip(os.sep)
- if os.access(filepath, os.X_OK):
- hooks[filepath] = name
- else:
- writemsg_level(" %s %s hook: '%s' is
not executable\n"
- % (warn("*"), _dir,
_unicode_decode(name),),
- level=logging.WARN,
noiselevel=2)
+ hooks = get_hooks_from_dir(_dir,
prefix=self.settings["PORTAGE_CONFIGROOT"])
self.hooks[_dir] = hooks
def __getattr__(self, name):
diff --git a/lib/portage/util/hooks.py b/lib/portage/util/hooks.py
new file mode 100644
index 0..d10ec7a59
--- /dev/null
+++ b/lib/portage/util/hooks.py
@@ -0,0 +1,31 @@
+# Copyright 2014-2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+
+from collections import OrderedDict
+
+import portage
+
+from portage import os
+from portage.output import create_color_func
+from portage.util import writemsg_level, _recursive_file_list
+from warnings import warn
+
+warn = create_color_func("WARN")
+
+
+def get_hooks_from_dir(rel_directory, prefix="/"):
+ directory = os.path.join(prefix, portage.USER_CONFIG_PATH,
rel_directory)
+
+ hooks = OrderedDict()
+ for filepath in _recursive_file_list(directory):
+ name = filepath.split(directory)[1].lstrip(portage.os.sep)
+ if portage.os.access(filepath, portage.os.X_OK):
+ hooks[filepath] = name
+ else:
+ writemsg_level(" %s %s hook: '%s' is not executable\n"
% \
+ (warn("*"), directory,
portage._unicode_decode(name),),
+ level=logging.WARN, noiselevel=2)
+
+ return hooks
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 21739ea1e0a793e207b7ae28d524e45582d9b2ca Author: Florian Schmaus geekplace eu> AuthorDate: Tue Apr 27 11:02:56 2021 + Commit: Zac Medico gentoo org> CommitDate: Mon May 24 06:13:07 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=21739ea1 env-update: skip PATH in systemd user environment definition Having PATH in the systemd user environment created by env-update makes it impossible to use "systemctl --user import-environment PATH" to set PATH in systemd user units according to the current value of PATH. Using "systemctl --user import-environment PATH" is a well known idiom, and hence should work. Therefore, we skip the creation of the PATH entry by env-update. Closes: https://github.com/gentoo/portage/pull/701 Signed-off-by: Florian Schmaus geekplace.eu> Signed-off-by: Zac Medico gentoo.org> lib/portage/util/env_update.py | 5 + 1 file changed, 5 insertions(+) diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py index 5588931a8..abf261fbc 100644 --- a/lib/portage/util/env_update.py +++ b/lib/portage/util/env_update.py @@ -367,6 +367,11 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, systemd_gentoo_env.write(senvnotice) for env_key in env_keys: + # Skip PATH since this makes it impossible to use + # "systemctl --user import-environment PATH". + if env_key == 'PATH': + continue + env_key_value = env[env_key] # Skip variables with the empty string
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: bfe550f69ac4e0126412bc83f7e6f6f0d1c6dc1f
Author: Zac Medico gentoo org>
AuthorDate: Sat May 1 23:38:50 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sat May 1 23:49:07 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bfe550f6
PipeLogger: handle FileNotFoundError when cancelled during _start
Bug: https://bugs.gentoo.org/787542
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_async/PipeLogger.py | 35 +--
1 file changed, 25 insertions(+), 10 deletions(-)
diff --git a/lib/portage/util/_async/PipeLogger.py
b/lib/portage/util/_async/PipeLogger.py
index b7c03043f..a335ce96f 100644
--- a/lib/portage/util/_async/PipeLogger.py
+++ b/lib/portage/util/_async/PipeLogger.py
@@ -33,16 +33,31 @@ class PipeLogger(AbstractPollTask):
self._log_file = log_file_path
_set_nonblocking(self._log_file.fileno())
elif log_file_path is not None:
- self._log_file = open(_unicode_encode(log_file_path,
- encoding=_encodings['fs'], errors='strict'),
mode='ab')
- if log_file_path.endswith('.gz'):
- self._log_file_real = self._log_file
- self._log_file = gzip.GzipFile(filename='',
mode='ab',
- fileobj=self._log_file)
-
- portage.util.apply_secpass_permissions(log_file_path,
- uid=portage.portage_uid,
gid=portage.portage_gid,
- mode=0o660)
+ try:
+ self._log_file = open(
+ _unicode_encode(
+ log_file_path,
encoding=_encodings["fs"], errors="strict"
+ ),
+ mode="ab",
+ )
+
+ if log_file_path.endswith(".gz"):
+ self._log_file_real = self._log_file
+ self._log_file = gzip.GzipFile(
+ filename="", mode="ab",
fileobj=self._log_file
+ )
+
+ portage.util.apply_secpass_permissions(
+ log_file_path,
+ uid=portage.portage_uid,
+ gid=portage.portage_gid,
+ mode=0o660,
+ )
+ except FileNotFoundError:
+ if self._was_cancelled():
+ self._async_wait()
+ return
+ raise
if isinstance(self.input_fd, int):
self.input_fd = os.fdopen(self.input_fd, 'rb', 0)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: b08e21e5a7e07da02a2d64dac80635ca82a12979
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 7 16:00:54 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 7 16:01:47 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b08e21e5
bin_entry_point: fix sys.argv[0] for non-python programs
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/bin_entry_point.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/portage/util/bin_entry_point.py
b/lib/portage/util/bin_entry_point.py
index 7d359052e..3ead95d98 100644
--- a/lib/portage/util/bin_entry_point.py
+++ b/lib/portage/util/bin_entry_point.py
@@ -28,6 +28,7 @@ def bin_entry_point():
script_path,
] + sys.argv[1:]
os.execvp(sys.argv[0], sys.argv)
+ sys.argv[0] = script_path
os.execvp(sys.argv[0], sys.argv)
else:
print("File not found:", script_path, file=sys.stderr)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: ce2247b4de0342d7802d4744904bb82870cac1ac Author: Zac Medico gentoo org> AuthorDate: Sun Mar 7 14:35:57 2021 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 7 14:37:54 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ce2247b4 PipeLogger: Use async and await syntax Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_async/PipeLogger.py | 12 +--- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/portage/util/_async/PipeLogger.py b/lib/portage/util/_async/PipeLogger.py index e8203268c..b7c03043f 100644 --- a/lib/portage/util/_async/PipeLogger.py +++ b/lib/portage/util/_async/PipeLogger.py @@ -1,4 +1,4 @@ -# Copyright 2008-2020 Gentoo Authors +# Copyright 2008-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import fcntl @@ -9,7 +9,6 @@ import portage from portage import os, _encodings, _unicode_encode from portage.util.futures import asyncio from portage.util.futures._asyncio.streams import _writer -from portage.util.futures.compat_coroutine import coroutine from portage.util.futures.unix_events import _set_nonblocking from _emerge.AbstractPollTask import AbstractPollTask @@ -53,7 +52,7 @@ class PipeLogger(AbstractPollTask): fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) - self._io_loop_task = asyncio.ensure_future(self._io_loop(self.input_fd, loop=self.scheduler), loop=self.scheduler) + self._io_loop_task = asyncio.ensure_future(self._io_loop(self.input_fd), loop=self.scheduler) self._io_loop_task.add_done_callback(self._io_loop_done) self._registered = True @@ -62,8 +61,7 @@ class PipeLogger(AbstractPollTask): if self.returncode is None: self.returncode = self._cancelled_returncode - @coroutine - def _io_loop(self, input_file, loop=None): + async def _io_loop(self, input_file): background = self.background stdout_fd = self.stdout_fd log_file = self._log_file @@ -77,7 +75,7 @@ class PipeLogger(AbstractPollTask): future = self.scheduler.create_future() self.scheduler.add_reader(fd, future.set_result, None) try: - yield future + await future finally: # The loop and input file may have been closed. if not self.scheduler.is_closed(): @@ -130,7 +128,7 @@ class PipeLogger(AbstractPollTask): if self._log_file_nb: # Use the _writer function which uses os.write, since the # log_file.write method looses data when an EAGAIN occurs. - yield _writer(log_file, buf, loop=self.scheduler) + await _writer(log_file, buf) else: # For gzip.GzipFile instances, the above _writer function # will not work because data written directly to the file
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: 03b4af7b288589d0fa475af2a2c9f66022203649 Author: Zac Medico gentoo org> AuthorDate: Sun Mar 7 14:43:59 2021 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 7 14:44:16 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=03b4af7b BuildLogger: Use async and await syntax Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_async/BuildLogger.py | 12 +--- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/portage/util/_async/BuildLogger.py b/lib/portage/util/_async/BuildLogger.py index 5a9c076b6..896e4d7b5 100644 --- a/lib/portage/util/_async/BuildLogger.py +++ b/lib/portage/util/_async/BuildLogger.py @@ -1,4 +1,4 @@ -# Copyright 2020 Gentoo Authors +# Copyright 2020-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import functools @@ -11,7 +11,6 @@ from portage.util import shlex_split from portage.util._async.PipeLogger import PipeLogger from portage.util._async.PopenProcess import PopenProcess from portage.util.futures import asyncio -from portage.util.futures.compat_coroutine import coroutine class BuildLogger(AsynchronousTask): """ @@ -78,16 +77,15 @@ class BuildLogger(AsynchronousTask): pipe_logger.start() self._main_task_cancel = functools.partial(self._main_cancel, filter_proc, pipe_logger) - self._main_task = asyncio.ensure_future(self._main(filter_proc, pipe_logger, loop=self.scheduler), loop=self.scheduler) + self._main_task = asyncio.ensure_future(self._main(filter_proc, pipe_logger), loop=self.scheduler) self._main_task.add_done_callback(self._main_exit) - @coroutine - def _main(self, filter_proc, pipe_logger, loop=None): + async def _main(self, filter_proc, pipe_logger): try: if pipe_logger.poll() is None: - yield pipe_logger.async_wait() + await pipe_logger.async_wait() if filter_proc is not None and filter_proc.poll() is None: - yield filter_proc.async_wait() + await filter_proc.async_wait() except asyncio.CancelledError: self._main_cancel(filter_proc, pipe_logger) raise
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: 9eea2af22c9e51475a4adba57fdded3a2a88c886 Author: Zac Medico gentoo org> AuthorDate: Sun Mar 7 14:18:44 2021 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 7 14:37:46 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9eea2af2 _writer: Use async and await syntax Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/_asyncio/streams.py | 13 + 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/portage/util/futures/_asyncio/streams.py b/lib/portage/util/futures/_asyncio/streams.py index ea5882dd3..7a8d4a3e0 100644 --- a/lib/portage/util/futures/_asyncio/streams.py +++ b/lib/portage/util/futures/_asyncio/streams.py @@ -1,4 +1,4 @@ -# Copyright 2018-2020 Gentoo Authors +# Copyright 2018-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -9,7 +9,6 @@ portage.proxy.lazyimport.lazyimport(globals(), '_emerge.PipeReader:PipeReader', 'portage.util.futures:asyncio', ) -from portage.util.futures.compat_coroutine import coroutine def _reader(input_file, loop=None): @@ -55,8 +54,7 @@ class _Reader: self._pipe_reader = None -@coroutine -def _writer(output_file, content, loop=None): +async def _writer(output_file, content, loop=DeprecationWarning): """ Asynchronously write bytes to output file. The output file is assumed to be in non-blocking mode. If an EnvironmentError @@ -68,10 +66,9 @@ def _writer(output_file, content, loop=None): @type output_file: file object @param content: content to write @type content: bytes - @param loop: asyncio.AbstractEventLoop (or compatible) - @type loop: event loop + @param loop: deprecated """ - loop = asyncio._wrap_loop(loop) + loop = asyncio.get_event_loop() fd = output_file.fileno() while content: try: @@ -82,7 +79,7 @@ def _writer(output_file, content, loop=None): waiter = loop.create_future() loop.add_writer(fd, lambda: waiter.done() or waiter.set_result(None)) try: - yield waiter + await waiter finally: # The loop and output file may have been closed. if not loop.is_closed():
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/
commit: e0b6fbdf6543c19c63bbca007278d48003b807ad Author: Zac Medico gentoo org> AuthorDate: Sun Mar 7 09:28:08 2021 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 7 09:28:37 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e0b6fbdf AsyncioEventLoop: remove obsolete supports_multiprocessing attribute Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_eventloop/asyncio_event_loop.py | 7 +-- 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index b77728088..6dfac3569 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -1,4 +1,4 @@ -# Copyright 2018-2020 Gentoo Authors +# Copyright 2018-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os @@ -17,11 +17,6 @@ class AsyncioEventLoop(_AbstractEventLoop): event loop and is minimally compatible with _PortageEventLoop. """ - # Use portage's internal event loop in subprocesses, as a workaround - # for https://bugs.python.org/issue22087, and also - # https://bugs.python.org/issue29703 which affects pypy3-5.10.1. - supports_multiprocessing = False - def __init__(self, loop=None): loop = loop or _real_asyncio.get_event_loop() self._loop = loop
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/
commit: 20204fd8c29f3060da9891879721a54486247b0c
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 7 07:44:49 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 7 07:45:11 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=20204fd8
Remove unused _PortageEventLoop and _PortageChildWatcher
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/unix_events.py | 112
1 file changed, 112 deletions(-)
diff --git a/lib/portage/util/futures/unix_events.py
b/lib/portage/util/futures/unix_events.py
index 030070b1b..4feee0a3b 100644
--- a/lib/portage/util/futures/unix_events.py
+++ b/lib/portage/util/futures/unix_events.py
@@ -18,61 +18,6 @@ from portage.util._eventloop.global_event_loop import (
)
-class _PortageEventLoop(events.AbstractEventLoop):
- """
- Implementation of asyncio.AbstractEventLoop which wraps portage's
- internal event loop.
- """
-
- def __init__(self, loop):
- """
- @type loop: EventLoop
- @param loop: an instance of portage's internal event loop
- """
- self._loop = loop
- self.run_until_complete = loop.run_until_complete
- self.call_soon = loop.call_soon
- self.call_soon_threadsafe = loop.call_soon_threadsafe
- self.call_later = loop.call_later
- self.call_at = loop.call_at
- self.is_running = loop.is_running
- self.is_closed = loop.is_closed
- self.close = loop.close
- self.create_future = loop.create_future
- self.add_reader = loop.add_reader
- self.remove_reader = loop.remove_reader
- self.add_writer = loop.add_writer
- self.remove_writer = loop.remove_writer
- self.run_in_executor = loop.run_in_executor
- self.time = loop.time
- self.default_exception_handler = loop.default_exception_handler
- self.call_exception_handler = loop.call_exception_handler
- self.set_debug = loop.set_debug
- self.get_debug = loop.get_debug
-
- @property
- def _asyncio_child_watcher(self):
- """
- In order to avoid accessing the internal _loop attribute,
portage
- internals should use this property when possible.
-
- @rtype: asyncio.AbstractChildWatcher
- @return: the internal event loop's AbstractChildWatcher
interface
- """
- return self._loop._asyncio_child_watcher
-
- @property
- def _asyncio_wrapper(self):
- """
- In order to avoid accessing the internal _loop attribute,
portage
- internals should use this property when possible.
-
- @rtype: asyncio.AbstractEventLoop
- @return: the internal event loop's AbstractEventLoop interface
- """
- return self
-
-
if hasattr(os, 'set_blocking'):
def _set_nonblocking(fd):
os.set_blocking(fd, False)
@@ -83,63 +28,6 @@ else:
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
-class _PortageChildWatcher(AbstractChildWatcher):
- def __init__(self, loop):
- """
- @type loop: EventLoop
- @param loop: an instance of portage's internal event loop
- """
- self._loop = loop
- self._callbacks = {}
-
- def close(self):
- pass
-
- def __enter__(self):
- return self
-
- def __exit__(self, a, b, c):
- pass
-
- def _child_exit(self, pid, status, data):
- self._callbacks.pop(pid)
- callback, args = data
- callback(pid, self._compute_returncode(status), *args)
-
- def _compute_returncode(self, status):
- if os.WIFSIGNALED(status):
- return -os.WTERMSIG(status)
- if os.WIFEXITED(status):
- return os.WEXITSTATUS(status)
- return status
-
- def add_child_handler(self, pid, callback, *args):
- """
- Register a new child handler.
-
- Arrange for callback(pid, returncode, *args) to be called when
- process 'pid' terminates. Specifying another callback for the
same
- process replaces the previous handler.
- """
- source_id = self._callbacks.get(pid)
- if source_id is not None:
- self._loop.source_remove(source_id)
- self._callbacks[pid] = self._loop.child_watch_add(
- pid, self._child_exit, data=(callback, args))
-
- def remove_child_handler(self, pid):
- """
- Removes the handler for
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/
commit: b4f6a5126437afc5bb07ed0a42a0eb8a5745ef90 Author: Zac Medico gentoo org> AuthorDate: Sun Mar 7 07:18:58 2021 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 7 07:32:04 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b4f6a512 Removed unused portage.util.futures.events Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/events.py | 186 lib/portage/util/futures/unix_events.py | 26 + 2 files changed, 3 insertions(+), 209 deletions(-) diff --git a/lib/portage/util/futures/events.py b/lib/portage/util/futures/events.py deleted file mode 100644 index 85032fcdf..0 --- a/lib/portage/util/futures/events.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2018 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -__all__ = ( - 'AbstractEventLoopPolicy', - 'AbstractEventLoop', -) - -import socket -import subprocess - -from asyncio.events import ( - AbstractEventLoop as _AbstractEventLoop, - AbstractEventLoopPolicy as _AbstractEventLoopPolicy, -) - - -class AbstractEventLoopPolicy(_AbstractEventLoopPolicy): - """Abstract policy for accessing the event loop.""" - - def get_event_loop(self): - raise NotImplementedError - - def set_event_loop(self, loop): - raise NotImplementedError - - def new_event_loop(self): - raise NotImplementedError - - def get_child_watcher(self): - raise NotImplementedError - - def set_child_watcher(self, watcher): - raise NotImplementedError - - -class AbstractEventLoop(_AbstractEventLoop): - """Abstract event loop.""" - - def run_forever(self): - raise NotImplementedError - - def run_until_complete(self, future): - raise NotImplementedError - - def stop(self): - raise NotImplementedError - - def is_running(self): - raise NotImplementedError - - def is_closed(self): - raise NotImplementedError - - def close(self): - raise NotImplementedError - - def shutdown_asyncgens(self): - raise NotImplementedError - - def _timer_handle_cancelled(self, handle): - raise NotImplementedError - - def call_soon(self, callback, *args): - return self.call_later(0, callback, *args) - - def call_later(self, delay, callback, *args): - raise NotImplementedError - - def call_at(self, when, callback, *args): - raise NotImplementedError - - def time(self): - raise NotImplementedError - - def create_future(self): - raise NotImplementedError - - def create_task(self, coro): - raise NotImplementedError - - def call_soon_threadsafe(self, callback, *args): - raise NotImplementedError - - def run_in_executor(self, executor, func, *args): - raise NotImplementedError - - def set_default_executor(self, executor): - raise NotImplementedError - - def getaddrinfo(self, host, port, family=0, type=0, proto=0, flags=0): # pylint: disable=redefined-builtin - raise NotImplementedError - - def getnameinfo(self, sockaddr, flags=0): - raise NotImplementedError - - def create_connection(self, protocol_factory, host=None, port=None, - ssl=None, family=0, proto=0, flags=0, sock=None, - local_addr=None, server_hostname=None): - raise NotImplementedError - - def create_server(self, protocol_factory, host=None, port=None, - family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, - sock=None, backlog=100, ssl=None, reuse_address=None, - reuse_port=None): - raise NotImplementedError - - def create_unix_connection(self, protocol_factory, path, - ssl=None, sock=None, - server_hostname=None): - raise NotImplementedError - - def create_unix_server(self, protocol_factory, path, - sock=None, backlog=100, ssl=None): - raise NotImplementedError - - def create_datagram_endpoint(self, protocol_factory, - local_addr=None, remote_addr=None, -family=0, proto=0, flags=0, - reuse_address=None, reuse_port=None, -
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/
commit: a8e0ddccf90a4f6dd0b4c0ae0832b4abb1f35b04
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 7 07:29:35 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 7 07:31:11 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a8e0ddcc
Remove unused _EventLoopFuture class
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/futures.py | 156 +---
1 file changed, 1 insertion(+), 155 deletions(-)
diff --git a/lib/portage/util/futures/futures.py
b/lib/portage/util/futures/futures.py
index 839c767a7..3f239890a 100644
--- a/lib/portage/util/futures/futures.py
+++ b/lib/portage/util/futures/futures.py
@@ -1,4 +1,4 @@
-# Copyright 2016-2018 Gentoo Foundation
+# Copyright 2016-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
#
# For compatibility with python versions which do not have the
@@ -19,157 +19,3 @@ from asyncio import (
InvalidStateError,
TimeoutError,
)
-# pylint: enable=redefined-builtin
-
-import portage
-portage.proxy.lazyimport.lazyimport(globals(),
-
'portage.util._eventloop.global_event_loop:global_event_loop@_global_event_loop',
-)
-
-_PENDING = 'PENDING'
-_CANCELLED = 'CANCELLED'
-_FINISHED = 'FINISHED'
-
-class _EventLoopFuture:
- """
- This class provides (a subset of) the asyncio.Future interface, for
- use with the EventLoop class, because EventLoop is currently
- missing some of the asyncio.AbstractEventLoop methods that
- asyncio.Future requires.
- """
-
- # Class variables serving as defaults for instance variables.
- _state = _PENDING
- _result = None
- _exception = None
- _loop = None
-
- def __init__(self, loop=None):
- """Initialize the future.
-
- The optional loop argument allows explicitly setting the event
- loop object used by the future. If it's not provided, the
future uses
- the default event loop.
- """
- if loop is None:
- self._loop = _global_event_loop()
- else:
- self._loop = loop
- self._callbacks = []
-
- def cancel(self):
- """Cancel the future and schedule callbacks.
-
- If the future is already done or cancelled, return False.
Otherwise,
- change the future's state to cancelled, schedule the callbacks
and
- return True.
- """
- if self._state != _PENDING:
- return False
- self._state = _CANCELLED
- self._schedule_callbacks()
- return True
-
- def _schedule_callbacks(self):
- """Internal: Ask the event loop to call all callbacks.
-
- The callbacks are scheduled to be called as soon as possible.
Also
- clears the callback list.
- """
- callbacks = self._callbacks[:]
- if not callbacks:
- return
-
- self._callbacks[:] = []
- for callback in callbacks:
- self._loop.call_soon(callback, self)
-
- def cancelled(self):
- """Return True if the future was cancelled."""
- return self._state == _CANCELLED
-
- def done(self):
- """Return True if the future is done.
-
- Done means either that a result / exception are available, or
that the
- future was cancelled.
- """
- return self._state != _PENDING
-
- def result(self):
- """Return the result this future represents.
-
- If the future has been cancelled, raises CancelledError. If the
- future's result isn't yet available, raises InvalidStateError.
If
- the future is done and has an exception set, this exception is
raised.
- """
- if self._state == _CANCELLED:
- raise CancelledError()
- if self._state != _FINISHED:
- raise InvalidStateError('Result is not ready.')
- if self._exception is not None:
- raise self._exception
- return self._result
-
- def exception(self):
- """Return the exception that was set on this future.
-
- The exception (or None if no exception was set) is returned
only if
- the future is done. If the future has been cancelled, raises
- CancelledError. If the future isn't done yet, raises
- InvalidStateError.
- """
- if self._state == _CANCELLED:
- raise CancelledError
- if self._state != _FINISHED:
- raise InvalidStateError('Exception is
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/
commit: ed685eb659fccf6e4031d12fa8a59c3829ef1155
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 7 06:56:36 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 7 07:01:44 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ed685eb6
Removed unused portage.util.futures.transports
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/transports.py | 87 --
lib/portage/util/futures/unix_events.py | 492 +---
2 files changed, 2 insertions(+), 577 deletions(-)
diff --git a/lib/portage/util/futures/transports.py
b/lib/portage/util/futures/transports.py
deleted file mode 100644
index 016ecbef8..0
--- a/lib/portage/util/futures/transports.py
+++ /dev/null
@@ -1,87 +0,0 @@
-# Copyright 2018 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-from asyncio.transports import Transport as _Transport
-
-
-class _FlowControlMixin(_Transport):
- """
- This is identical to the standard library's private
- asyncio.transports._FlowControlMixin class.
-
- All the logic for (write) flow control in a mix-in base class.
-
- The subclass must implement get_write_buffer_size(). It must call
- _maybe_pause_protocol() whenever the write buffer size increases,
- and _maybe_resume_protocol() whenever it decreases. It may also
- override set_write_buffer_limits() (e.g. to specify different
- defaults).
-
- The subclass constructor must call super().__init__(extra). This
- will call set_write_buffer_limits().
-
- The user may call set_write_buffer_limits() and
- get_write_buffer_size(), and their protocol's pause_writing() and
- resume_writing() may be called.
- """
-
- def __init__(self, extra=None, loop=None):
- super().__init__(extra)
- assert loop is not None
- self._loop = loop
- self._protocol_paused = False
- self._set_write_buffer_limits()
-
- def _maybe_pause_protocol(self):
- size = self.get_write_buffer_size()
- if size <= self._high_water:
- return
- if not self._protocol_paused:
- self._protocol_paused = True
- try:
- self._protocol.pause_writing()
- except Exception as exc:
- self._loop.call_exception_handler({
- 'message': 'protocol.pause_writing()
failed',
- 'exception': exc,
- 'transport': self,
- 'protocol': self._protocol,
- })
-
- def _maybe_resume_protocol(self):
- if (self._protocol_paused and
- self.get_write_buffer_size() <= self._low_water):
- self._protocol_paused = False
- try:
- self._protocol.resume_writing()
- except Exception as exc:
- self._loop.call_exception_handler({
- 'message': 'protocol.resume_writing()
failed',
- 'exception': exc,
- 'transport': self,
- 'protocol': self._protocol,
- })
-
- def get_write_buffer_limits(self):
- return (self._low_water, self._high_water)
-
- def _set_write_buffer_limits(self, high=None, low=None):
- if high is None:
- if low is None:
- high = 64*1024
- else:
- high = 4*low
- if low is None:
- low = high // 4
- if not high >= low >= 0:
- raise ValueError('high (%r) must be >= low (%r) must be
>= 0' %
-(high, low))
- self._high_water = high
- self._low_water = low
-
- def set_write_buffer_limits(self, high=None, low=None):
- self._set_write_buffer_limits(high=high, low=low)
- self._maybe_pause_protocol()
-
- def get_write_buffer_size(self):
- raise NotImplementedError
diff --git a/lib/portage/util/futures/unix_events.py
b/lib/portage/util/futures/unix_events.py
index 16a9e12b7..9d5445943 100644
--- a/lib/portage/util/futures/unix_events.py
+++ b/lib/portage/util/futures/unix_events.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = (
@@ -7,32 +7,15 @@ __all__ = (
)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: 15049a041909f85b02a52f5b1938c6dd4171c9e3
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 7 03:03:40 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 7 05:12:48 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=15049a04
Removed unused portage.util.futures._asyncio.tasks
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/_asyncio/__init__.py | 46 ++---
lib/portage/util/futures/_asyncio/tasks.py| 96 ---
2 files changed, 19 insertions(+), 123 deletions(-)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index 207e7205d..4643697e0 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -25,7 +25,16 @@ import types
import weakref
import asyncio as _real_asyncio
-from asyncio.subprocess import Process
+# pylint: disable=redefined-builtin
+from asyncio import (
+ ALL_COMPLETED,
+ CancelledError,
+ FIRST_COMPLETED,
+ FIRST_EXCEPTION,
+ Future,
+ InvalidStateError,
+ TimeoutError,
+)
try:
import threading
@@ -38,20 +47,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures:compat_coroutine@_compat_coroutine',
)
from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as
_AsyncioEventLoop
-# pylint: disable=redefined-builtin
-from portage.util.futures.futures import (
- CancelledError,
- Future,
- InvalidStateError,
- TimeoutError,
-)
-# pylint: enable=redefined-builtin
-from portage.util.futures._asyncio.tasks import (
- ALL_COMPLETED,
- FIRST_COMPLETED,
- FIRST_EXCEPTION,
- wait,
-)
_lock = threading.Lock()
@@ -131,20 +126,17 @@ def create_subprocess_exec(*args, **kwargs):
# Python 3.4 and later implement PEP 446, which makes newly
# created file descriptors non-inheritable by default.
kwargs.setdefault('close_fds', False)
- if isinstance(loop._asyncio_wrapper, _AsyncioEventLoop):
- # Use the real asyncio create_subprocess_exec (loop argument
- # is deprecated since since Python 3.8).
- return _real_asyncio.create_subprocess_exec(*args, **kwargs)
-
- result = loop.create_future()
+ # Use the real asyncio create_subprocess_exec (loop argument
+ # is deprecated since since Python 3.8).
+ return ensure_future(_real_asyncio.create_subprocess_exec(*args,
**kwargs), loop=loop)
- result.set_result(Process(subprocess.Popen(
- args,
- stdin=kwargs.pop('stdin', None),
- stdout=kwargs.pop('stdout', None),
- stderr=kwargs.pop('stderr', None), **kwargs), loop))
- return result
+def wait(futures, loop=None, timeout=None, return_when=ALL_COMPLETED):
+ """
+ Wraps asyncio.wait() and omits the loop argument which is not
+ supported since python 3.10.
+ """
+ return _real_asyncio.wait(futures, timeout=timeout,
return_when=return_when)
def iscoroutinefunction(func):
diff --git a/lib/portage/util/futures/_asyncio/tasks.py
b/lib/portage/util/futures/_asyncio/tasks.py
deleted file mode 100644
index c9db3146e..0
--- a/lib/portage/util/futures/_asyncio/tasks.py
+++ /dev/null
@@ -1,96 +0,0 @@
-# Copyright 2018-2020 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-___all___ = (
- 'ALL_COMPLETED',
- 'FIRST_COMPLETED',
- 'FIRST_EXCEPTION',
- 'wait',
-)
-
-from asyncio import ALL_COMPLETED, FIRST_COMPLETED, FIRST_EXCEPTION
-
-import portage
-portage.proxy.lazyimport.lazyimport(globals(),
- 'portage.util.futures:asyncio',
-)
-
-def wait(futures, loop=None, timeout=None, return_when=ALL_COMPLETED):
- """
- Use portage's internal EventLoop to emulate asyncio.wait:
- https://docs.python.org/3/library/asyncio-task.html#asyncio.wait
-
- @param futures: futures to wait for
- @type futures: asyncio.Future (or compatible)
- @param timeout: number of seconds to wait (wait indefinitely if
- not specified)
- @type timeout: int or float
- @param return_when: indicates when this function should return, must
- be one of the constants ALL_COMPLETED, FIRST_COMPLETED, or
- FIRST_EXCEPTION (default is ALL_COMPLETED)
- @type return_when: object
- @param loop: event loop
- @type loop: EventLoop
- @return: tuple of (done, pending).
- @rtype: asyncio.Future (or compatible)
- """
- loop = asyncio._wrap_loop(loop)
- result_future = loop.create_future()
- _Waiter(futures, timeout, return_when, result_future, loop)
- return result_future
-
-
-class _Waiter:
- def __init__(self, futures, timeout, return_when, result_future, loop):
-
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/
commit: 4cf221840bb980305b48a329c693dd1a310d5248
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 7 02:48:57 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 7 02:49:34 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4cf22184
Remove unused PollSelectAdapter and PollConstants classes
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_eventloop/PollConstants.py | 17 --
lib/portage/util/_eventloop/PollSelectAdapter.py | 74
2 files changed, 91 deletions(-)
diff --git a/lib/portage/util/_eventloop/PollConstants.py
b/lib/portage/util/_eventloop/PollConstants.py
deleted file mode 100644
index c5700d108..0
--- a/lib/portage/util/_eventloop/PollConstants.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright 1999-2009 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-import select
-class PollConstants:
-
- """
- Provides POLL* constants that are equivalent to those from the
- select module, for use by PollSelectAdapter.
- """
-
- names = ("POLLIN", "POLLPRI", "POLLOUT", "POLLERR", "POLLHUP",
"POLLNVAL")
- v = 1
- for k in names:
- locals()[k] = getattr(select, k, v)
- v *= 2
- del k, v
diff --git a/lib/portage/util/_eventloop/PollSelectAdapter.py
b/lib/portage/util/_eventloop/PollSelectAdapter.py
deleted file mode 100644
index c4637a352..0
--- a/lib/portage/util/_eventloop/PollSelectAdapter.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# Copyright 1999-2020 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-import select
-
-from .PollConstants import PollConstants
-
-class PollSelectAdapter:
-
- """
- Use select to emulate a poll object, for
- systems that don't support poll().
- """
-
- def __init__(self):
- self._registered = {}
- self._select_args = [[], [], []]
-
- def register(self, fd, *args):
- """
- Only POLLIN is currently supported!
- """
- if len(args) > 1:
- raise TypeError(
- "register expected at most 2 arguments, got " +
\
- repr(1 + len(args)))
-
- eventmask = PollConstants.POLLIN | \
- PollConstants.POLLPRI | PollConstants.POLLOUT
- if args:
- eventmask = args[0]
-
- self._registered[fd] = eventmask
- self._select_args = None
-
- def unregister(self, fd):
- self._select_args = None
- del self._registered[fd]
-
- def poll(self, *args):
- if len(args) > 1:
- raise TypeError(
- "poll expected at most 2 arguments, got " + \
- repr(1 + len(args)))
-
- timeout = None
- if args:
- timeout = args[0]
-
- select_args = self._select_args
- if select_args is None:
- select_args = [list(self._registered), [], []]
-
- if timeout is not None:
- select_args = select_args[:]
- # Translate poll() timeout args to select() timeout
args:
- #
- # | units| value(s) for indefinite
block
- #
-|--|--
- # poll | milliseconds | omitted, negative, or None
- #
-|--|--
- # select | seconds | omitted
- #
-|--|--
-
- if timeout is not None and timeout < 0:
- timeout = None
- if timeout is not None:
- select_args.append(timeout / 1000)
-
- select_events = select.select(*select_args)
- poll_events = []
- for fd in select_events[0]:
- poll_events.append((fd, PollConstants.POLLIN))
- return poll_events
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 4f60b68fca342cc92ed7729d241e2c60132876e8 Author: Zac Medico gentoo org> AuthorDate: Sat Mar 6 20:20:30 2021 + Commit: Zac Medico gentoo org> CommitDate: Sat Mar 6 20:20:46 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4f60b68f bin_entry_point: handle versioned python shebangs Signed-off-by: Zac Medico gentoo.org> lib/portage/util/bin_entry_point.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/util/bin_entry_point.py b/lib/portage/util/bin_entry_point.py index ce95231eb..7d359052e 100644 --- a/lib/portage/util/bin_entry_point.py +++ b/lib/portage/util/bin_entry_point.py @@ -20,7 +20,7 @@ def bin_entry_point(): if os.access(script_path, os.X_OK): with open(script_path, "rt") as f: shebang = f.readline() - python_match = re.search(r"/python\s+([^/]*)\s+$", shebang) + python_match = re.search(r"/python[\d\.]*\s+([^/]*)\s+$", shebang) if python_match: sys.argv = [ os.path.join(os.path.dirname(sys.argv[0]), "python"),
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/
commit: 05e9b76523569f17c3cbf7465758656c3beb0be8
Author: Zac Medico gentoo org>
AuthorDate: Sat Mar 6 10:39:31 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Mar 6 10:41:06 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=05e9b765
Remove unused EventLoop class
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_eventloop/EventLoop.py | 1153 --
1 file changed, 1153 deletions(-)
diff --git a/lib/portage/util/_eventloop/EventLoop.py
b/lib/portage/util/_eventloop/EventLoop.py
deleted file mode 100644
index ff2b73255..0
--- a/lib/portage/util/_eventloop/EventLoop.py
+++ /dev/null
@@ -1,1153 +0,0 @@
-# Copyright 1999-2020 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-import asyncio as _real_asyncio
-import collections
-import errno
-import functools
-import logging
-import os
-import select
-import signal
-import time
-import traceback
-
-try:
- import fcntl
-except ImportError:
- # http://bugs.jython.org/issue1074
- fcntl = None
-
-try:
- import threading
-except ImportError:
- import dummy_threading as threading
-
-import portage
-portage.proxy.lazyimport.lazyimport(globals(),
- 'portage.util.futures:asyncio',
- 'portage.util.futures.executor.fork:ForkExecutor',
-
'portage.util.futures.unix_events:_PortageEventLoop,_PortageChildWatcher',
-)
-
-from portage.util import writemsg_level
-from ..SlotObject import SlotObject
-from .PollConstants import PollConstants
-from .PollSelectAdapter import PollSelectAdapter
-
-class EventLoop:
- """
- An event loop, intended to be compatible with the GLib event loop.
- Call the iteration method in order to execute one iteration of the
- loop. The idle_add and timeout_add methods serve as thread-safe
- means to interact with the loop's thread.
- """
-
- supports_multiprocessing = True
-
- # TODO: Find out why SIGCHLD signals aren't delivered during poll
- # calls, forcing us to wakeup in order to receive them.
- _sigchld_interval = 250
-
- class _child_callback_class(SlotObject):
- __slots__ = ("callback", "data", "pid", "source_id")
-
- class _idle_callback_class(SlotObject):
- __slots__ = ("_args", "_callback", "_cancelled")
-
- class _io_handler_class(SlotObject):
- __slots__ = ("args", "callback", "f", "source_id")
-
- class _timeout_handler_class(SlotObject):
- __slots__ = ("args", "function", "calling", "interval",
"source_id",
- "timestamp")
-
- class _handle:
- """
- A callback wrapper object, compatible with asyncio.Handle.
- """
- __slots__ = ("_callback_id", "_loop")
-
- def __init__(self, callback_id, loop):
- self._callback_id = callback_id
- self._loop = loop
-
- def cancel(self):
- """
- Cancel the call. If the callback is already canceled or
executed,
- this method has no effect.
- """
- self._loop.source_remove(self._callback_id)
-
- class _call_soon_callback:
- """
- Wraps a call_soon callback, and always returns False, since
these
- callbacks are only supposed to run once.
- """
- __slots__ = ("_args", "_callback")
-
- def __init__(self, callback, args):
- self._callback = callback
- self._args = args
-
- def __call__(self):
- self._callback(*self._args)
- return False
-
- class _selector_callback:
- """
- Wraps an callback, and always returns True, for callbacks that
- are supposed to run repeatedly.
- """
- __slots__ = ("_args", "_callbacks")
-
- def __init__(self, callbacks):
- self._callbacks = callbacks
-
- def __call__(self, fd, event):
- for callback, mask in self._callbacks:
- if event & mask:
- callback()
- return True
-
- def __init__(self, main=True):
- """
- @param main: If True then this is a singleton instance for use
- in the main thread, otherwise it is a local instance
which
- can safely be use in a non-main thread (default is
True, so
- that global_event_loop does not need constructor
arguments)
- @type main: bool
- """
- self._use_signal = main and fcntl is not None
-
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/
commit: 1e843f853a9afe82d599e6ab09064147ddc1d271
Author: Matt Turner gentoo org>
AuthorDate: Thu Mar 4 19:24:58 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Mar 6 09:06:53 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1e843f85
Use asyncio.subprocess.Process directly
With no need to support Python 2, we can remove our private
implementation.
Signed-off-by: Matt Turner gentoo.org>
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/_asyncio/__init__.py | 8 +-
lib/portage/util/futures/_asyncio/process.py | 116 --
2 files changed, 4 insertions(+), 120 deletions(-)
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index 5590963f1..207e7205d 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -25,6 +25,7 @@ import types
import weakref
import asyncio as _real_asyncio
+from asyncio.subprocess import Process
try:
import threading
@@ -45,7 +46,6 @@ from portage.util.futures.futures import (
TimeoutError,
)
# pylint: enable=redefined-builtin
-from portage.util.futures._asyncio.process import _Process
from portage.util.futures._asyncio.tasks import (
ALL_COMPLETED,
FIRST_COMPLETED,
@@ -124,8 +124,8 @@ def create_subprocess_exec(*args, **kwargs):
@type loop: event loop
@type kwargs: varies
@param kwargs: subprocess.Popen parameters
- @rtype: asyncio.Future (or compatible)
- @return: subset of asyncio.subprocess.Process interface
+ @rtype: asyncio.subprocess.Process (or compatible)
+ @return: asyncio.subprocess.Process interface
"""
loop = _wrap_loop(kwargs.pop('loop', None))
# Python 3.4 and later implement PEP 446, which makes newly
@@ -138,7 +138,7 @@ def create_subprocess_exec(*args, **kwargs):
result = loop.create_future()
- result.set_result(_Process(subprocess.Popen(
+ result.set_result(Process(subprocess.Popen(
args,
stdin=kwargs.pop('stdin', None),
stdout=kwargs.pop('stdout', None),
diff --git a/lib/portage/util/futures/_asyncio/process.py
b/lib/portage/util/futures/_asyncio/process.py
deleted file mode 100644
index 275c9031a..0
--- a/lib/portage/util/futures/_asyncio/process.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# Copyright 2018-2020 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-import os
-
-import portage
-portage.proxy.lazyimport.lazyimport(globals(),
- 'portage.util.futures:asyncio',
- 'portage.util.futures.unix_events:_set_nonblocking',
-)
-from portage.util.futures._asyncio.streams import _reader, _writer
-from portage.util.futures.compat_coroutine import coroutine, coroutine_return
-
-
-class _Process:
- """
- Emulate a subset of the asyncio.subprocess.Process interface,
- for python2.
- """
- def __init__(self, proc, loop):
- """
- @param proc: process instance
- @type proc: subprocess.Popen
- @param loop: asyncio.AbstractEventLoop (or compatible)
- @type loop: event loop
- """
- self._proc = proc
- self._loop = loop
- self.terminate = proc.terminate
- self.kill = proc.kill
- self.send_signal = proc.send_signal
- self.pid = proc.pid
- self._waiters = []
- loop._asyncio_child_watcher.\
- add_child_handler(self.pid, self._proc_exit)
-
- @property
- def returncode(self):
- return self._proc.returncode
-
- @coroutine
- def communicate(self, input=None, loop=None): # pylint:
disable=redefined-builtin
- """
- Read data from stdout and stderr, until end-of-file is reached.
- Wait for process to terminate.
-
- @param input: stdin content to write
- @type input: bytes
- @return: tuple (stdout_data, stderr_data)
- @rtype: asyncio.Future (or compatible)
- """
- loop = asyncio._wrap_loop(loop or self._loop)
- futures = []
- for input_file in (self._proc.stdout, self._proc.stderr):
- if input_file is None:
- future = loop.create_future()
- future.set_result(None)
- else:
- future = _reader(input_file, loop=loop)
- futures.append(future)
-
- writer = None
- if input is not None:
- if self._proc.stdin is None:
- raise TypeError('communicate: expected file or
int, got
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: ce86ddecf168af06926f95092f29fa19c1f3885a
Author: Zac Medico gentoo org>
AuthorDate: Wed Mar 3 10:46:02 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Mar 3 12:09:58 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ce86ddec
bin_entry_point: rewrite python shebangs for venv
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/bin_entry_point.py | 12 +++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/lib/portage/util/bin_entry_point.py
b/lib/portage/util/bin_entry_point.py
index 7f2ee3849..ce95231eb 100644
--- a/lib/portage/util/bin_entry_point.py
+++ b/lib/portage/util/bin_entry_point.py
@@ -3,6 +3,7 @@
__all__ = ["bin_entry_point"]
+import re
import sys
from portage.const import PORTAGE_BIN_PATH
@@ -17,7 +18,16 @@ def bin_entry_point():
"""
script_path = os.path.join(PORTAGE_BIN_PATH,
os.path.basename(sys.argv[0]))
if os.access(script_path, os.X_OK):
- sys.argv[0] = script_path
+ with open(script_path, "rt") as f:
+ shebang = f.readline()
+ python_match = re.search(r"/python\s+([^/]*)\s+$", shebang)
+ if python_match:
+ sys.argv = [
+ os.path.join(os.path.dirname(sys.argv[0]),
"python"),
+ python_match.group(1),
+ script_path,
+ ] + sys.argv[1:]
+ os.execvp(sys.argv[0], sys.argv)
os.execvp(sys.argv[0], sys.argv)
else:
print("File not found:", script_path, file=sys.stderr)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, /
commit: 695be424d3ef3bba6cf26f6ff70df066e1cd248c
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 28 07:34:13 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Feb 28 09:49:05 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=695be424
setup.py: add entry_points when installed with pip
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/bin_entry_point.py | 24
setup.py| 27 +--
2 files changed, 45 insertions(+), 6 deletions(-)
diff --git a/lib/portage/util/bin_entry_point.py
b/lib/portage/util/bin_entry_point.py
new file mode 100644
index 0..7f2ee3849
--- /dev/null
+++ b/lib/portage/util/bin_entry_point.py
@@ -0,0 +1,24 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+__all__ = ["bin_entry_point"]
+
+import sys
+
+from portage.const import PORTAGE_BIN_PATH
+from portage import os
+
+
+def bin_entry_point():
+ """
+ Adjust sys.argv[0] to point to a script in PORTAGE_BIN_PATH, and
+ then execute the script, in order to implement entry_points when
+ portage has been installed by pip.
+ """
+ script_path = os.path.join(PORTAGE_BIN_PATH,
os.path.basename(sys.argv[0]))
+ if os.access(script_path, os.X_OK):
+ sys.argv[0] = script_path
+ os.execvp(sys.argv[0], sys.argv)
+ else:
+ print("File not found:", script_path, file=sys.stderr)
+ return 127
diff --git a/setup.py b/setup.py
index 96d9932e0..7d6fb6609 100755
--- a/setup.py
+++ b/setup.py
@@ -2,6 +2,7 @@
# Copyright 1998-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from distutils import sysconfig
from distutils.core import setup, Command, Extension
from distutils.command.build import build
from distutils.command.build_ext import build_ext as _build_ext
@@ -19,6 +20,7 @@ from distutils.util import change_root, subst_vars
import codecs
import collections
import glob
+import itertools
import os
import os.path
import platform
@@ -26,6 +28,12 @@ import re
import subprocess
import sys
+autodetect_pip = os.path.basename(os.environ.get("_", "")) == "pip" or
os.path.basename(
+ os.path.dirname(__file__)
+).startswith("pip-")
+venv_prefix = "" if sys.prefix == sys.base_prefix else sys.prefix
+create_entry_points = bool(autodetect_pip or venv_prefix)
+eprefix = sysconfig.get_python_lib() if venv_prefix else ""
# TODO:
# - smarter rebuilds of docs w/ 'install_docbook' and 'install_apidoc'.
@@ -220,8 +228,9 @@ class x_build_scripts_custom(build_scripts):
self.scripts = x_scripts[self.dir_name]
else:
self.scripts = set(self.scripts)
- for other_files in x_scripts.values():
-
self.scripts.difference_update(other_files)
+ if not (create_entry_points and self.dir_name
== "portage"):
+ for other_files in x_scripts.values():
+
self.scripts.difference_update(other_files)
def run(self):
# group scripts by subdirectory
@@ -471,9 +480,10 @@ class x_install_lib(install_lib):
'VERSION': self.distribution.get_version(),
})
rewrite_file('portage/const.py', {
- 'PORTAGE_BASE_PATH': self.portage_base,
- 'PORTAGE_BIN_PATH': self.portage_bindir,
- 'PORTAGE_CONFIG_PATH': self.portage_confdir,
+ 'EPREFIX': eprefix,
+ 'GLOBAL_CONFIG_PATH': self.portage_confdir,
+ 'PORTAGE_BASE_PATH': eprefix + self.portage_base,
+ 'PORTAGE_BIN_PATH': eprefix + self.portage_bindir,
})
return ret
@@ -675,7 +685,12 @@ setup(
['$portage_base/bin', ['bin/deprecated-path']],
['$sysconfdir/portage/repo.postsync.d',
['cnf/repo.postsync.d/example']],
],
-
+ entry_points={
+ "console_scripts": [
+
"{}=portage.util.bin_entry_point:bin_entry_point".format(os.path.basename(path))
+ for path in
itertools.chain.from_iterable(x_scripts.values())
+ ],
+ } if create_entry_points else {},
ext_modules = [Extension(name=n, sources=m,
extra_compile_args=['-D_FILE_OFFSET_BITS=64',
'-D_LARGEFILE_SOURCE', '-D_LARGEFILE64_SOURCE'])
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 061a895f7b0a310b374dff03af622b11146337d4 Author: Zac Medico gentoo org> AuthorDate: Thu Feb 25 10:14:18 2021 + Commit: Zac Medico gentoo org> CommitDate: Thu Feb 25 10:18:02 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=061a895f iter_parents: prevent infinite loop after empty os.path.dirname result Reported-by: dkjii cock.li Bug: https://bugs.gentoo.org/772806 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/path.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/portage/util/path.py b/lib/portage/util/path.py index a0b96c7f3..f174bd71f 100644 --- a/lib/portage/util/path.py +++ b/lib/portage/util/path.py @@ -1,4 +1,4 @@ -# Copyright 2014 Gentoo Foundation +# Copyright 2014-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -45,4 +45,6 @@ def iter_parents(path): yield path while path != os.sep: path = os.path.dirname(path) + if not path: + break yield path
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/, bin/
commit: d121ea57ed5310d84328be27f10a8556b0a7d7ba
Author: Zac Medico gentoo org>
AuthorDate: Fri May 8 23:32:49 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 24 15:27:47 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d121ea57
Add emirrordist shelve dump/restore (bug 721680)
Bug: https://bugs.gentoo.org/721680
Signed-off-by: Zac Medico gentoo.org>
bin/shelve-utils | 32 +++
lib/portage/tests/util/test_shelve.py | 60 +++
lib/portage/util/shelve.py| 58 +
3 files changed, 150 insertions(+)
diff --git a/bin/shelve-utils b/bin/shelve-utils
new file mode 100755
index 0..5088ab5eb
--- /dev/null
+++ b/bin/shelve-utils
@@ -0,0 +1,32 @@
+#!/usr/bin/python -b
+# Copyright 2020-2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import argparse
+import sys
+
+import portage
+from portage.util.shelve import dump, restore
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(prog="shelve-utils")
+ subparsers = parser.add_subparsers(help="sub-command help")
+
+ dump_command = subparsers.add_parser("dump", help="dump shelve
database")
+ dump_command.add_argument("src", help="input shelve file")
+ dump_command.add_argument("dest", help="output pickle file")
+ dump_command.set_defaults(func=dump)
+
+ restore_command = subparsers.add_parser("restore", help="restore shelve
database")
+ restore_command.add_argument("src", help="input pickle file")
+ restore_command.add_argument("dest", help="output shelve file")
+ restore_command.set_defaults(func=restore)
+
+ args = parser.parse_args(args=portage._decode_argv(argv or
sys.argv)[1:])
+ args.func(args)
+
+
+if __name__ == "__main__":
+ portage.util.initialize_logger()
+ main(argv=sys.argv)
diff --git a/lib/portage/tests/util/test_shelve.py
b/lib/portage/tests/util/test_shelve.py
new file mode 100644
index 0..60592c6fb
--- /dev/null
+++ b/lib/portage/tests/util/test_shelve.py
@@ -0,0 +1,60 @@
+# Copyright 2020-2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import argparse
+import os
+import shutil
+import tempfile
+import time
+
+from portage.tests import TestCase
+from portage.util.shelve import dump, open_shelve, restore
+
+
+class ShelveUtilsTestCase(TestCase):
+
+ TEST_DATA = (
+ # distfiles_db
+ {
+ "portage-2.3.89.tar.bz2": "sys-apps/portage-2.3.89",
+ "portage-2.3.99.tar.bz2": "sys-apps/portage-2.3.99",
+ },
+ # deletion_db
+ {
+ "portage-2.3.89.tar.bz2": time.time(),
+ "portage-2.3.99.tar.bz2": time.time(),
+ },
+ # recycle_db
+ {
+ "portage-2.3.89.tar.bz2": (0, time.time()),
+ "portage-2.3.99.tar.bz2": (0, time.time()),
+ },
+ )
+
+ def test_dump_restore(self):
+ for data in self.TEST_DATA:
+ tmpdir = tempfile.mkdtemp()
+ try:
+ dump_args = argparse.Namespace(
+ src=os.path.join(tmpdir, "shelve_file"),
+ dest=os.path.join(tmpdir,
"pickle_file"),
+ )
+ db = open_shelve(dump_args.src, flag="c")
+ for k, v in data.items():
+ db[k] = v
+ db.close()
+ dump(dump_args)
+
+ os.unlink(dump_args.src)
+ restore_args = argparse.Namespace(
+ dest=dump_args.src,
+ src=dump_args.dest,
+ )
+ restore(restore_args)
+
+ db = open_shelve(restore_args.dest, flag="r")
+ for k, v in data.items():
+ self.assertEqual(db[k], v)
+ db.close()
+ finally:
+ shutil.rmtree(tmpdir)
diff --git a/lib/portage/util/shelve.py b/lib/portage/util/shelve.py
new file mode 100644
index 0..f070ee753
--- /dev/null
+++ b/lib/portage/util/shelve.py
@@ -0,0 +1,58 @@
+# Copyright 2020-2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import pickle
+import shelve
+
+
+def open_shelve(db_file, flag="r"):
+ """
+ The optional flag parameter has the same interpretation as the flag
+
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_async/
commit: becd2d4f82ab212fd752848d673a72722af209ff Author: Zac Medico gentoo org> AuthorDate: Mon Feb 15 04:28:40 2021 + Commit: Zac Medico gentoo org> CommitDate: Mon Feb 15 04:35:56 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=becd2d4f PopenProcess: use call_soon for _async_waipid in _start Use call_soon to delay the add_child_handler call that _async_waipid will trigger, so that it will occur after the event loop is running. Bug: https://bugs.gentoo.org/770712 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_async/PopenProcess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/util/_async/PopenProcess.py b/lib/portage/util/_async/PopenProcess.py index c1931327a..7f4e17ea2 100644 --- a/lib/portage/util/_async/PopenProcess.py +++ b/lib/portage/util/_async/PopenProcess.py @@ -1,4 +1,4 @@ -# Copyright 2012-2018 Gentoo Foundation +# Copyright 2012-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from _emerge.SubProcess import SubProcess @@ -13,7 +13,7 @@ class PopenProcess(SubProcess): self._registered = True if self.pipe_reader is None: - self._async_waitpid() + self.scheduler.call_soon(self._async_waitpid) else: try: self.pipe_reader.scheduler = self.scheduler
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 4f77b1ca80a7ae7987c0d3e0ef988b3c06d18744 Author: Zac Medico gentoo org> AuthorDate: Tue Jan 19 00:50:13 2021 + Commit: Zac Medico gentoo org> CommitDate: Tue Jan 19 00:50:26 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4f77b1ca ProxyManager: remove unused loop parameter Signed-off-by: Zac Medico gentoo.org> lib/portage/util/socks5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index ddf6bb4d0..4aa08e1ab 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -73,7 +73,7 @@ class ProxyManager: """ return self.socket_path is not None - async def ready(self, loop=None): + async def ready(self): """ Wait for the proxy socket to become ready. This method is a coroutine. """
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/, lib/portage/repository/storage/
commit: 5346ae96d8d4b141454fe802a74e855e3e246593
Author: Zac Medico gentoo org>
AuthorDate: Mon Jan 18 14:37:17 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jan 18 15:03:28 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5346ae96
RepoStorageInterface: Use async and await syntax
Signed-off-by: Zac Medico gentoo.org>
.../repository/storage/hardlink_quarantine.py | 45 +++--
lib/portage/repository/storage/hardlink_rcu.py | 57 +-
lib/portage/repository/storage/inplace.py | 27 +++---
lib/portage/repository/storage/interface.py| 17 +++
lib/portage/util/futures/_sync_decorator.py| 6 +--
5 files changed, 56 insertions(+), 96 deletions(-)
diff --git a/lib/portage/repository/storage/hardlink_quarantine.py
b/lib/portage/repository/storage/hardlink_quarantine.py
index 165ab8324..bab34997b 100644
--- a/lib/portage/repository/storage/hardlink_quarantine.py
+++ b/lib/portage/repository/storage/hardlink_quarantine.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage import os
@@ -7,10 +7,6 @@ from portage.repository.storage.interface import (
RepoStorageInterface,
)
from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import (
- coroutine,
- coroutine_return,
-)
from _emerge.SpawnProcess import SpawnProcess
@@ -38,60 +34,55 @@ class HardlinkQuarantineRepoStorage(RepoStorageInterface):
self._spawn_kwargs = spawn_kwargs
self._current_update = None
- @coroutine
- def _check_call(self, cmd, loop=None):
+ async def _check_call(self, cmd):
"""
Run cmd and raise RepoStorageException on failure.
@param cmd: command to executre
@type cmd: list
"""
- p = SpawnProcess(args=cmd, scheduler=asyncio._wrap_loop(loop),
**self._spawn_kwargs)
+ p = SpawnProcess(args=cmd, scheduler=asyncio.get_event_loop(),
**self._spawn_kwargs)
p.start()
- if (yield p.async_wait()) != os.EX_OK:
+ if await p.async_wait() != os.EX_OK:
raise RepoStorageException('command exited with status
{}: {}'.\
format(p.returncode, ' '.join(cmd)))
- @coroutine
- def init_update(self, loop=None):
+ async def init_update(self):
update_location = os.path.join(self._user_location,
'.tmp-unverified-download-quarantine')
- yield self._check_call(['rm', '-rf', update_location],
loop=loop)
+ await self._check_call(['rm', '-rf', update_location])
# Use rsync --link-dest to hardlink a files into
self._update_location,
# since cp -l is not portable.
- yield self._check_call(['rsync', '-a', '--link-dest',
self._user_location,
+ await self._check_call(['rsync', '-a', '--link-dest',
self._user_location,
'--exclude=/distfiles', '--exclude=/local',
'--exclude=/lost+found', '--exclude=/packages',
'--exclude',
'/{}'.format(os.path.basename(update_location)),
- self._user_location + '/', update_location + '/'],
loop=loop)
+ self._user_location + '/', update_location + '/'])
self._update_location = update_location
- coroutine_return(self._update_location)
+ return self._update_location
@property
- def current_update(self, loop=None):
+ def current_update(self):
if self._update_location is None:
raise RepoStorageException('current update does not
exist')
return self._update_location
- @coroutine
- def commit_update(self, loop=None):
+ async def commit_update(self):
update_location = self.current_update
self._update_location = None
- yield self._check_call(['rsync', '-a', '--delete',
+ await self._check_call(['rsync', '-a', '--delete',
'--exclude=/distfiles', '--exclude=/local',
'--exclude=/lost+found', '--exclude=/packages',
'--exclude',
'/{}'.format(os.path.basename(update_location)),
- update_location + '/', self._user_location + '/'],
loop=loop)
+ update_location + '/', self._user_location + '/'])
- yield self._check_call(['rm', '-rf', update_location],
loop=loop)
+ await self._check_call(['rm', '-rf', update_location])
- @coroutine
- def abort_update(self, loop=None):
+ async def abort_update(self):
if
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/
commit: 8d962cb5cc97a5092ff45446c0f8da55b27d2434
Author: Zac Medico gentoo org>
AuthorDate: Mon Jan 18 10:09:05 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jan 18 11:18:43 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8d962cb5
coroutine: do not require loop parameter
The loop parameter is not needed since global_event_loop now
returns the running event loop for the current thread.
Bug: https://bugs.gentoo.org/737698
Bug: https://bugs.gentoo.org/763339
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/futures/compat_coroutine.py | 6 +-
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/lib/portage/util/futures/compat_coroutine.py
b/lib/portage/util/futures/compat_coroutine.py
index 9a0c5c1c8..3e8dcec02 100644
--- a/lib/portage/util/futures/compat_coroutine.py
+++ b/lib/portage/util/futures/compat_coroutine.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2021 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -68,10 +68,6 @@ def _generator_future(generator_func, *args, **kwargs):
the default event loop.
"""
loop = kwargs.get('loop')
- if loop is None and portage._internal_caller:
- # Require an explicit loop parameter, in order to support
- # local event loops (bug 737698).
- raise AssertionError("Missing required argument 'loop'")
loop = asyncio._wrap_loop(loop)
result = loop.create_future()
_GeneratorTask(generator_func(*args, **kwargs), result, loop=loop)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/, lib/portage/util/futures/_asyncio/
commit: 386178481eb86ac603cd90ef1bb6ac6b68e51c50
Author: Zac Medico gentoo org>
AuthorDate: Mon Jan 4 09:14:36 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jan 11 09:36:48 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=38617848
global_event_loop: return running loop for current thread
Like asyncio.get_event_loop(), return the running loop for the
current thread if there is one, and otherwise construct a new
one if needed. This allows the _safe_loop function to become
synonymous with the global_event_loop function.
For the case of "loop running in non-main thread" of API
consumer, this change makes portage compatible with PEP
492 coroutines with async and await syntax. Portage
internals can safely begin using async / await syntax instead
of compat_coroutine.
Bug: https://bugs.gentoo.org/763339
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/_eventloop/global_event_loop.py | 28 +++---
lib/portage/util/futures/_asyncio/__init__.py| 30 +---
2 files changed, 24 insertions(+), 34 deletions(-)
diff --git a/lib/portage/util/_eventloop/global_event_loop.py
b/lib/portage/util/_eventloop/global_event_loop.py
index 413011178..cb7a13078 100644
--- a/lib/portage/util/_eventloop/global_event_loop.py
+++ b/lib/portage/util/_eventloop/global_event_loop.py
@@ -1,28 +1,6 @@
-# Copyright 2012-2020 Gentoo Authors
+# Copyright 2012-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import portage
-from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop
+__all__ = ('global_event_loop',)
-_instances = {}
-
-
-def global_event_loop():
- """
- Get a global EventLoop (or compatible object) instance which
- belongs exclusively to the current process.
- """
-
- pid = portage.getpid()
- instance = _instances.get(pid)
- if instance is not None:
- return instance
-
- constructor = AsyncioEventLoop
-
- # Use the _asyncio_wrapper attribute, so that unit tests can compare
- # the reference to one retured from _wrap_loop(), since they should
- # not close the loop if it refers to a global event loop.
- instance = constructor()._asyncio_wrapper
- _instances[pid] = instance
- return instance
+from portage.util.futures._asyncio import _safe_loop as global_event_loop
diff --git a/lib/portage/util/futures/_asyncio/__init__.py
b/lib/portage/util/futures/_asyncio/__init__.py
index d39f31786..5590963f1 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018-2020 Gentoo Authors
+# Copyright 2018-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = (
@@ -37,9 +37,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures:compat_coroutine@_compat_coroutine',
)
from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as
_AsyncioEventLoop
-from portage.util._eventloop.global_event_loop import (
- global_event_loop as _global_event_loop,
-)
# pylint: disable=redefined-builtin
from portage.util.futures.futures import (
CancelledError,
@@ -238,7 +235,7 @@ def _wrap_loop(loop=None):
# The default loop returned by _wrap_loop should be consistent
# with global_event_loop, in order to avoid accidental registration
# of callbacks with a loop that is not intended to run.
- loop = loop or _global_event_loop()
+ loop = loop or _safe_loop()
return (loop if hasattr(loop, '_asyncio_wrapper')
else _AsyncioEventLoop(loop=loop))
@@ -267,13 +264,15 @@ def _safe_loop():
@rtype: asyncio.AbstractEventLoop (or compatible)
@return: event loop instance
"""
- if portage._internal_caller or threading.current_thread() is
threading.main_thread():
- return _global_event_loop()
+ loop = _get_running_loop()
+ if loop is not None:
+ return loop
thread_key = threading.get_ident()
with _thread_weakrefs.lock:
if _thread_weakrefs.pid != portage.getpid():
_thread_weakrefs.pid = portage.getpid()
+ _thread_weakrefs.mainloop = None
_thread_weakrefs.loops = weakref.WeakValueDictionary()
try:
loop = _thread_weakrefs.loops[thread_key]
@@ -283,9 +282,23 @@ def _safe_loop():
except RuntimeError:
_real_asyncio.set_event_loop(_real_asyncio.new_event_loop())
loop = _thread_weakrefs.loops[thread_key] =
_AsyncioEventLoop()
+
+ if _thread_weakrefs.mainloop is None and threading.current_thread() is
threading.main_thread():
+ _thread_weakrefs.mainloop = loop
+
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/
commit: eeeb70bf50994c08ebcddac94474105f0635360c Author: Zac Medico gentoo org> AuthorDate: Mon Jan 11 06:46:49 2021 + Commit: Zac Medico gentoo org> CommitDate: Mon Jan 11 07:28:28 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=eeeb70bf AsyncioEventLoop: wrap child watcher for thread safety (bug 764905) Use a child watcher wrapper to deliver the callbacks via the call_soon_threadsafe method, since documentation for the asycio AbstractChildWatcher class says that callbacks must be thread safe. Bug: https://bugs.gentoo.org/764905 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/_eventloop/asyncio_event_loop.py | 30 ++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index 4d7047ae8..b77728088 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -6,6 +6,7 @@ import signal import asyncio as _real_asyncio from asyncio.events import AbstractEventLoop as _AbstractEventLoop +from asyncio.unix_events import AbstractChildWatcher as _AbstractChildWatcher import portage @@ -47,6 +48,7 @@ class AsyncioEventLoop(_AbstractEventLoop): self.set_debug = loop.set_debug self.get_debug = loop.get_debug self._wakeup_fd = -1 + self._child_watcher = None if portage._internal_caller: loop.set_exception_handler(self._internal_caller_exception_handler) @@ -87,7 +89,9 @@ class AsyncioEventLoop(_AbstractEventLoop): @rtype: asyncio.AbstractChildWatcher @return: the internal event loop's AbstractChildWatcher interface """ - return _real_asyncio.get_child_watcher() + if self._child_watcher is None: + self._child_watcher = _ChildWatcherThreadSafetyWrapper(self, _real_asyncio.get_child_watcher()) + return self._child_watcher @property def _asyncio_wrapper(self): @@ -126,3 +130,27 @@ class AsyncioEventLoop(_AbstractEventLoop): except ValueError: # This is intended to fail when not called in the main thread. pass + + +class _ChildWatcherThreadSafetyWrapper(_AbstractChildWatcher): + def __init__(self, loop, real_watcher): + self._loop = loop + self._real_watcher = real_watcher + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, a, b, c): + pass + + def _child_exit(self, pid, status, callback, *args): + self._loop.call_soon_threadsafe(callback, pid, status, *args) + + def add_child_handler(self, pid, callback, *args): + self._real_watcher.add_child_handler(pid, self._child_exit, callback, *args) + + def remove_child_handler(self, pid): + return self._real_watcher.remove_child_handler(pid)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/
commit: 7efa7ecfe07737239be593b2c32e497cc1d2f154 Author: Zac Medico gentoo org> AuthorDate: Mon Jan 4 08:54:52 2021 + Commit: Zac Medico gentoo org> CommitDate: Mon Jan 4 08:57:09 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7efa7ecf _Retry: Use ensure_future for self._current_task Use ensure_future for compatibility with PEP 492 coroutines with async and await syntax. Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/retry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/util/futures/retry.py b/lib/portage/util/futures/retry.py index 4092f60d6..31cc161da 100644 --- a/lib/portage/util/futures/retry.py +++ b/lib/portage/util/futures/retry.py @@ -1,4 +1,4 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -113,7 +113,7 @@ class _Retry: def _begin_try(self): self._tries += 1 - self._current_task = self._func() + self._current_task = asyncio.ensure_future(self._func(), loop=self._loop) self._current_task.add_done_callback(self._try_done) if self._try_timeout is not None: self._try_timeout_handle = self._loop.call_later(
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 84b555bb33f24b7628bebbaa16fe43325b7823a9 Author: Zac Medico gentoo org> AuthorDate: Sat Jan 2 01:18:30 2021 + Commit: Zac Medico gentoo org> CommitDate: Sat Jan 2 01:18:43 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=84b555bb lib/portage/util/__init__.py: Fix useless-return Signed-off-by: Zac Medico gentoo.org> lib/portage/util/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py index bedcbcfe6..b0922bf09 100644 --- a/lib/portage/util/__init__.py +++ b/lib/portage/util/__init__.py @@ -1293,7 +1293,6 @@ class atomic_ofstream(AbstractContextManager, ObjectProxy): self.abort() else: self.close() - return None def _get_target(self): return object.__getattribute__(self, '_file')
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: 1574ae127b270739c4293271c959d1d981684906
Author: Florian Schmaus geekplace eu>
AuthorDate: Fri Dec 18 18:46:39 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Dec 31 01:43:22 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1574ae12
env_update: use "with statement" on atomic_ofstream
Signed-off-by: Florian Schmaus geekplace.eu>
Signed-off-by: Zac Medico gentoo.org>
lib/portage/util/env_update.py | 30 --
1 file changed, 12 insertions(+), 18 deletions(-)
diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py
index dec086cf8..5588931a8 100644
--- a/lib/portage/util/env_update.py
+++ b/lib/portage/util/env_update.py
@@ -342,18 +342,17 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env,
#create /etc/profile.env for bash support
profile_env_path = os.path.join(eroot, "etc", "profile.env")
- outfile = atomic_ofstream(profile_env_path)
- outfile.write(penvnotice)
-
- env_keys = [x for x in env if x != "LDPATH"]
- env_keys.sort()
- for k in env_keys:
- v = env[k]
- if v.startswith('$') and not v.startswith('${'):
- outfile.write("export %s=$'%s'\n" % (k, v[1:]))
- else:
- outfile.write("export %s='%s'\n" % (k, v))
- outfile.close()
+ with atomic_ofstream(profile_env_path) as outfile:
+ outfile.write(penvnotice)
+
+ env_keys = [x for x in env if x != "LDPATH"]
+ env_keys.sort()
+ for k in env_keys:
+ v = env[k]
+ if v.startswith('$') and not v.startswith('${'):
+ outfile.write("export %s=$'%s'\n" % (k, v[1:]))
+ else:
+ outfile.write("export %s='%s'\n" % (k, v))
# Create the systemd user environment configuration file
# /etc/environment.d/10-gentoo-env.conf with the
@@ -363,8 +362,7 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env,
systemd_gentoo_env_path = os.path.join(systemd_environment_dir,
"10-gentoo-env.conf")
- systemd_gentoo_env = atomic_ofstream(systemd_gentoo_env_path)
- try:
+ with atomic_ofstream(systemd_gentoo_env_path) as systemd_gentoo_env:
senvnotice = notice + "\n\n"
systemd_gentoo_env.write(senvnotice)
@@ -384,10 +382,6 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env,
line = f"{env_key}={env_key_value}\n"
systemd_gentoo_env.write(line)
- except:
- systemd_gentoo_env.abort()
- raise
- systemd_gentoo_env.close()
#create /etc/csh.env for (t)csh support
outfile = atomic_ofstream(os.path.join(eroot, "etc", "csh.env"))
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: e93e6d65fa1ca75f676a227f7918f8b6d747425c Author: Florian Schmaus geekplace eu> AuthorDate: Fri Dec 18 18:46:38 2020 + Commit: Zac Medico gentoo org> CommitDate: Thu Dec 31 01:40:42 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e93e6d65 Make atomic_ofstream a Context Manager This allows using a "with statement" together with instances of atomic_ofstream. Allowing for more readable, less error prone and shorter code. Signed-off-by: Florian Schmaus geekplace.eu> Signed-off-by: Zac Medico gentoo.org> lib/portage/util/__init__.py | 10 +- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py index 0412b2b59..bedcbcfe6 100644 --- a/lib/portage/util/__init__.py +++ b/lib/portage/util/__init__.py @@ -11,6 +11,7 @@ __all__ = ['apply_permissions', 'apply_recursive_permissions', 'stack_dicts', 'stack_lists', 'unique_array', 'unique_everseen', 'varexpand', 'write_atomic', 'writedict', 'writemsg', 'writemsg_level', 'writemsg_stdout'] +from contextlib import AbstractContextManager from copy import deepcopy import errno import io @@ -1246,7 +1247,7 @@ def apply_secpass_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, stat_cached=stat_cached, follow_links=follow_links) return all_applied -class atomic_ofstream(ObjectProxy): +class atomic_ofstream(AbstractContextManager, ObjectProxy): """Write a file atomically via os.rename(). Atomic replacement prevents interprocess interference and prevents corruption of the target file when the write is interrupted (for example, when an 'out of space' @@ -1287,6 +1288,13 @@ class atomic_ofstream(ObjectProxy): encoding=_encodings['fs'], errors='strict'), mode=mode, **kargs)) + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + self.abort() + else: + self.close() + return None + def _get_target(self): return object.__getattribute__(self, '_file')
