[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, /, lib/portage/util/
commit: 4d17764863896903df4d18fe7c6b45635a18f6c4 Author: Ulrich Müller gentoo org> AuthorDate: Wed Jun 19 08:08:04 2024 + Commit: Ulrich Müller gentoo org> CommitDate: Wed Jun 19 18:17:06 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4d177648 vartree, movefile: Warn when rewriting a symlink See PMS section 13.4.1 (Rewriting): Any absolute symlink whose link starts with D must be rewritten with the leading D removed. The package manager should issue a notice when doing this. Bug: https://bugs.gentoo.org/934514 Signed-off-by: Ulrich Müller gentoo.org> NEWS | 2 ++ lib/portage/dbapi/vartree.py | 10 ++ lib/portage/util/movefile.py | 5 + 3 files changed, 17 insertions(+) diff --git a/NEWS b/NEWS index b4b378e7a0..d5a03533d9 100644 --- a/NEWS +++ b/NEWS @@ -20,6 +20,8 @@ Bug fixes: working ebuilds. Future EAPIs will need to adjust the logic added by this change. See bug #907061. +* vartree, movefile: Warn when rewriting a symlink (bug #934514). + portage-3.0.65 (2024-06-04) -- diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index beb1a6486f..0c41d408c3 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -5563,6 +5563,16 @@ class dblink: myabsto = myabsto.lstrip(sep) if self.settings and self.settings["D"]: if myto.startswith(self.settings["D"]): +self._eqawarn( +"preinst", +[ +_( +"QA Notice: Absolute symlink %s points to %s inside the image directory.\n" +"Removing the leading %s from its path." +) +% (mydest, myto, self.settings["D"]) +], +) myto = myto[len(self.settings["D"]) - 1 :] # myrealto contains the path of the real file to which this symlink points. # we can simply test for existence of this file to see if the target has been merged yet diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py index 75100a3acd..7b880d2e3e 100644 --- a/lib/portage/util/movefile.py +++ b/lib/portage/util/movefile.py @@ -210,6 +210,11 @@ def movefile( try: target = os.readlink(src) if mysettings and "D" in mysettings and target.startswith(mysettings["D"]): +writemsg( +f"!!! {_('Absolute symlink points to image directory.')}\n", +noiselevel=-1, +) +writemsg(f"!!! {dest} -> {target}\n", noiselevel=-1) target = target[len(mysettings["D"]) - 1 :] # Atomically update the path if it exists. try:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 1a7fc63d20ad2e1292be3697c105c2d7e1691f91 Author: Zac Medico gentoo org> AuthorDate: Sat Jun 1 04:56:45 2024 + Commit: Zac Medico gentoo org> CommitDate: Sat Jun 1 19:18:35 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1a7fc63d MergeProcess: Pass bintree to subprocess It's required for FEATURES=*-backup. Fixes: b9a85ff987ea ("MergeProcess: Support QueryCommand with spawn start method") Bug: https://bugs.gentoo.org/933297 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/_MergeProcess.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index d9ab2b47aa..34e39eb229 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -1,4 +1,4 @@ -# Copyright 2010-2023 Gentoo Authors +# Copyright 2010-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import functools @@ -183,11 +183,12 @@ class MergeProcess(ForkProcess): # Since the entire QueryCommand._db is not required, only pass # in tree types that QueryCommand specifically requires. +# NOTE: For FEATURES=*-backup bintree is needed (bug 933297). child_db = {} parent_db = portage.db if QueryCommand._db is None else QueryCommand._db for root in parent_db: child_db[root] = {} -for tree_type in ("vartree", "porttree"): +for tree_type in ("bintree", "porttree", "vartree"): child_db[root][tree_type] = parent_db[root][tree_type] self.target = functools.partial(
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 2db89a16ab87b85004216959ec6bc508575d96a0 Author: Zac Medico gentoo org> AuthorDate: Mon May 27 18:03:32 2024 + Commit: Zac Medico gentoo org> CommitDate: Mon May 27 18:04:41 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2db89a16 binarytree: Rewrite remote index only on change I noticed that the remote index was rewritten with a new DOWNLOAD_TIMESTAMP even while frozen, and this patch fixed it. Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 64dfee4faa..b32dea1eae 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1400,6 +1400,7 @@ class binarytree: except OSError as e: if e.errno != errno.ENOENT: raise +changed = True local_timestamp = pkgindex.header.get("TIMESTAMP", None) try: download_timestamp = float(pkgindex.header.get("DOWNLOAD_TIMESTAMP", 0)) @@ -1574,6 +1575,7 @@ class binarytree: noiselevel=-1, ) except UseCachedCopyOfRemoteIndex: +changed = False desc = "frozen" if repo.frozen else "up-to-date" writemsg_stdout("\n") writemsg_stdout( @@ -1611,7 +1613,7 @@ class binarytree: os.unlink(tmp_filename) except OSError: pass -if pkgindex is rmt_idx: +if pkgindex is rmt_idx and changed: pkgindex.modified = False # don't update the header pkgindex.header["DOWNLOAD_TIMESTAMP"] = "%d" % time.time() try:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/repository/, lib/portage/sync/modules/mercurial/, ...
commit: cdf328f5629c20ecf792a6200c8c2c24d932ee68 Author: Mike Gilbert gentoo org> AuthorDate: Mon May 13 19:21:47 2024 + Commit: Mike Gilbert gentoo org> CommitDate: Tue May 21 17:27:30 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cdf328f5 Drop portage.util.shlex_split This has been a trivial wrapper since we dropped support for python2. Signed-off-by: Mike Gilbert gentoo.org> bin/dispatch-conf | 5 +++-- bin/ebuild | 5 ++--- bin/egencache | 5 ++--- bin/quickpkg| 9 + lib/_emerge/BinpkgExtractorAsync.py | 14 +- lib/_emerge/BinpkgFetcher.py| 4 ++-- lib/_emerge/actions.py | 3 ++- lib/_emerge/main.py | 3 ++- lib/portage/_emirrordist/FetchTask.py | 3 ++- lib/portage/_sets/dbapi.py | 6 +++--- lib/portage/_sets/libs.py | 4 ++-- lib/portage/dbapi/bintree.py| 5 ++--- lib/portage/dbapi/porttree.py | 5 +++-- lib/portage/dbapi/vartree.py| 17 ++--- lib/portage/dispatch_conf.py| 5 +++-- lib/portage/emaint/modules/logs/logs.py | 5 +++-- lib/portage/emaint/modules/sync/sync.py | 3 ++- lib/portage/getbinpkg.py| 3 ++- lib/portage/gpg.py | 7 --- lib/portage/gpkg.py | 11 ++- lib/portage/package/ebuild/_config/LocationsManager.py | 4 ++-- .../package/ebuild/_config/env_var_validation.py| 4 ++-- .../package/ebuild/_parallel_manifest/ManifestTask.py | 5 +++-- lib/portage/package/ebuild/config.py| 6 +++--- lib/portage/package/ebuild/doebuild.py | 9 - lib/portage/package/ebuild/fetch.py | 6 +++--- lib/portage/repository/config.py| 4 ++-- lib/portage/sync/modules/git/git.py | 11 ++- lib/portage/sync/modules/mercurial/mercurial.py | 15 --- lib/portage/sync/modules/rsync/rsync.py | 9 +++-- lib/portage/tests/ebuild/test_fetch.py | 5 +++-- lib/portage/tests/emerge/conftest.py| 7 --- lib/portage/tests/emerge/test_config_protect.py | 5 +++-- lib/portage/util/ExtractKernelVersion.py| 5 +++-- lib/portage/util/__init__.py| 11 +-- lib/portage/util/_async/BuildLogger.py | 4 ++-- lib/portage/util/_dyn_libs/soname_deps.py | 6 +++--- 37 files changed, 116 insertions(+), 122 deletions(-) diff --git a/bin/dispatch-conf b/bin/dispatch-conf index e34e9587f7..93164d909e 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -14,6 +14,7 @@ import atexit import errno import re +import shlex import subprocess import sys import termios @@ -89,7 +90,7 @@ def cmd_var_is_valid(cmd): Return true if the first whitespace-separated token contained in cmd is an executable file, false otherwise. """ -cmd = portage.util.shlex_split(cmd) +cmd = shlex.split(cmd) if not cmd: return False @@ -130,7 +131,7 @@ class dispatch: if pager is None or not cmd_var_is_valid(pager): pager = "cat" -pager_basename = os.path.basename(portage.util.shlex_split(pager)[0]) +pager_basename = os.path.basename(shlex.split(pager)[0]) if pager_basename == "less": less_opts = self.options.get("less-opts") if less_opts is not None and less_opts.strip(): diff --git a/bin/ebuild b/bin/ebuild index 043e5bc476..113a6214d6 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -34,6 +34,7 @@ signal.signal(signal.SIGUSR1, debug_signal) import argparse from os import path as osp +import shlex import sys import textwrap @@ -107,9 +108,7 @@ def main(): parser.error("missing required args") if not opts.ignore_default_opts: -default_opts = portage.util.shlex_split( -portage.settings.get("EBUILD_DEFAULT_OPTS", "") -) +default_opts = shlex.split(portage.settings.get("EBUILD_DEFAULT_OPTS", "")) opts, pargs = parser.parse_known_args(default_opts + sys.argv[1:]) debug = opts.debug diff --git a/bin/egencache b/bin/egencache index dbe8d27fee..36477e1abf 100755 --- a/bin/egencache +++ b/bin/egencache @@ -33,6 +33,7 @@ try: signal.signal(signal.SIGUSR1, debug_signal) import argparse +import shlex import stat import sys import f
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/_emerge/, lib/portage/, bin/, ...
commit: a33065dab4bebd476d0dcba3c1659fbe0e515469 Author: Mike Gilbert gentoo org> AuthorDate: Wed May 15 17:25:18 2024 + Commit: Mike Gilbert gentoo org> CommitDate: Tue May 21 17:24:00 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a33065da Do not use shlex.split for CONFIG_PROTECT PMS says this is a whitespace-separated list, so we should not treat it as a shell expression. Signed-off-by: Mike Gilbert gentoo.org> bin/dispatch-conf | 4 ++-- bin/portageq| 8 lib/_emerge/depgraph.py | 6 +++--- lib/_emerge/post_emerge.py | 2 +- lib/portage/_global_updates.py | 6 +++--- lib/portage/dbapi/vartree.py| 8 lib/portage/emaint/modules/sync/sync.py | 4 +--- 7 files changed, 18 insertions(+), 20 deletions(-) diff --git a/bin/dispatch-conf b/bin/dispatch-conf index 601110ce87..e34e9587f7 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -202,7 +202,7 @@ class dispatch: protect_obj = portage.util.ConfigProtect( config_root, config_paths, - portage.util.shlex_split(portage.settings.get("CONFIG_PROTECT_MASK", "")), +portage.settings.get("CONFIG_PROTECT_MASK", "").split(), case_insensitive=("case-insensitive-fs" in portage.settings.features), ) @@ -616,4 +616,4 @@ if len(sys.argv) > 1: # for testing d.grind(sys.argv[1:]) else: -d.grind(portage.util.shlex_split(portage.settings.get("CONFIG_PROTECT", ""))) +d.grind(portage.settings.get("CONFIG_PROTECT", "").split()) diff --git a/bin/portageq b/bin/portageq index 93fa4edeba..9ef0cb7d62 100755 --- a/bin/portageq +++ b/bin/portageq @@ -410,8 +410,8 @@ try: from portage.util import ConfigProtect settings = portage.settings -protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", "")) -protect_mask = portage.util.shlex_split(settings.get("CONFIG_PROTECT_MASK", "")) +protect = settings.get("CONFIG_PROTECT", "").split() +protect_mask = settings.get("CONFIG_PROTECT_MASK", "").split() protect_obj = ConfigProtect( root, protect, @@ -449,8 +449,8 @@ try: from portage.util import ConfigProtect settings = portage.settings -protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", "")) -protect_mask = portage.util.shlex_split(settings.get("CONFIG_PROTECT_MASK", "")) +protect = settings.get("CONFIG_PROTECT", "").split() +protect_mask = settings.get("CONFIG_PROTECT_MASK", "").split() protect_obj = ConfigProtect( root, protect, diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 9673d85f87..13add990e6 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -55,7 +55,7 @@ from portage.package.ebuild.getmaskingstatus import _getmaskingstatus, _MaskReas from portage._sets import SETPREFIX from portage._sets.base import InternalPackageSet from portage.dep._slot_operator import evaluate_slot_operator_equal_deps -from portage.util import ConfigProtect, shlex_split, new_protect_filename +from portage.util import ConfigProtect, new_protect_filename from portage.util import cmp_sort_key, writemsg, writemsg_stdout from portage.util import ensure_dirs, normalize_path from portage.util import writemsg_level, write_atomic @@ -10650,8 +10650,8 @@ class depgraph: settings = self._frozen_config.roots[root].settings protect_obj[root] = ConfigProtect( settings["PORTAGE_CONFIGROOT"], -shlex_split(settings.get("CONFIG_PROTECT", "")), -shlex_split(settings.get("CONFIG_PROTECT_MASK", "")), +settings.get("CONFIG_PROTECT", "").split(), +settings.get("CONFIG_PROTECT_MASK", "").split(), case_insensitive=("case-insensitive-fs" in settings.features), ) diff --git a/lib/_emerge/post_emerge.py b/lib/_emerge/post_emerge.py index 37e2c3cc80..6f1f1c243d 100644 --- a/lib/_emerge/post_emerge.py +++ b/lib/_emerge/post_emerge.py @@ -93,7 +93,7 @@ def post_emerge(myaction, myopts, myfiles, target_root, trees, mtimedb, retval): settings.regenerate() settings.lock() -config_protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", "")) +config_protect = settings.get("CONFIG_PROTECT", "").split() infodirs = settings.get("INFOPATH", "").split(":") + settings.get( "INFODIR", "" ).split(":") diff --git a/lib/portage/_global_updates.py b/lib/portage/_global_updates.py index f7997fc37c..4ed8a3f9a7 100644 --- a/lib/portage/_global_updates.py +++ b/lib/portage/_global_updates.py @@ -15,7 +15,7 @@ from portage.update import ( update_config_files, update_dbentry, ) -from portage.util import
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 62ee9bf8c680b2a18713da5bd453e3a771257409 Author: Zac Medico gentoo org> AuthorDate: Sat Mar 2 22:48:54 2024 + Commit: Zac Medico gentoo org> CommitDate: Sat Mar 2 22:50:54 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=62ee9bf8 binarytree._populate_remote: Fix UnboundLocalError for binpkg-request-signature If an InvalidBinaryPackageFormat exception was raised from get_binpkg_format for binpkg-request-signature then it triggered an UnboundLocalError here. Fixes: 445f10f4214c ("Use binpkg extensions and header to get format") Bug: https://bugs.gentoo.org/926048 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index f4251b47d6..4ba1407cda 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1624,7 +1624,7 @@ class binarytree: binpkg_format = get_binpkg_format( d.get("PATH"), remote=True ) -except InvalidBinaryPackageFormat: +except InvalidBinaryPackageFormat as e: writemsg( colorize( "WARN",
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/, /, lib/portage/
commit: 19e27e0415fd321c39104f7d687bcdc4f4132e24 Author: Mike Gilbert gentoo org> AuthorDate: Sun Feb 25 18:10:15 2024 + Commit: Mike Gilbert gentoo org> CommitDate: Mon Feb 26 04:10:32 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=19e27e04 Add workaround for loading libc on musl musl libc has no soname, which causes ctypes.util.find_library to fail. As a fallback, try to load "libc.so". Signed-off-by: Mike Gilbert gentoo.org> NEWS| 7 +++ lib/portage/dbapi/_MergeProcess.py | 4 ++-- lib/portage/dbapi/_SyncfsProcess.py | 14 -- lib/portage/process.py | 16 +--- lib/portage/util/_ctypes.py | 15 +++ lib/portage/util/locale.py | 7 ++- 6 files changed, 35 insertions(+), 28 deletions(-) diff --git a/NEWS b/NEWS index eb84651b53..258d800373 100644 --- a/NEWS +++ b/NEWS @@ -6,8 +6,15 @@ Release notes take the form of the following optional categories: * Bug fixes * Cleanups +portage-3.0.64 (UNRELEASED) +-- + +Bug fixes: + + portage-3.0.63 (2024-02-25) -- +* ctypes: Add workaround for loading libc on musl Bug fixes: * emerge: Skip installed packages with emptytree in depgraph selection (bug #651018). diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index dd5ad71cf8..d9ab2b47aa 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -10,7 +10,7 @@ import fcntl import portage from portage import os, _unicode_decode from portage.package.ebuild._ipc.QueryCommand import QueryCommand -from portage.util._ctypes import find_library +from portage.util._ctypes import load_libc import portage.elog.messages from portage.util._async.ForkProcess import ForkProcess from portage.util import no_color @@ -64,7 +64,7 @@ class MergeProcess(ForkProcess): # process, so that it's only done once rather than # for each child process. if platform.system() == "Linux" and "merge-sync" in settings.features: -find_library("c") +load_libc() # Inherit stdin by default, so that the pdb SIGUSR1 # handler is usable for the subprocess. diff --git a/lib/portage/dbapi/_SyncfsProcess.py b/lib/portage/dbapi/_SyncfsProcess.py index ddc2240071..300ae53985 100644 --- a/lib/portage/dbapi/_SyncfsProcess.py +++ b/lib/portage/dbapi/_SyncfsProcess.py @@ -4,7 +4,7 @@ import functools from portage import os -from portage.util._ctypes import find_library, LoadLibrary +from portage.util._ctypes import load_libc from portage.util._async.ForkProcess import ForkProcess @@ -24,15 +24,9 @@ class SyncfsProcess(ForkProcess): @staticmethod def _get_syncfs(): -filename = find_library("c") -if filename is not None: -library = LoadLibrary(filename) -if library is not None: -try: -return library.syncfs -except AttributeError: -pass - +(libc, _) = load_libc() +if libc is not None: +return getattr(libc, "syncfs", None) return None @staticmethod diff --git a/lib/portage/process.py b/lib/portage/process.py index cc9ed7bf78..b652e32942 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -37,7 +37,7 @@ portage.proxy.lazyimport.lazyimport( from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY from portage.exception import CommandNotFound from portage.proxy.objectproxy import ObjectProxy -from portage.util._ctypes import find_library, LoadLibrary, ctypes +from portage.util._ctypes import load_libc, LoadLibrary, ctypes try: from portage.util.netlink import RtNetlink @@ -960,11 +960,9 @@ def _exec( have_unshare = False libc = None if unshare_net or unshare_ipc or unshare_mount or unshare_pid: -filename = find_library("c") -if filename is not None: -libc = LoadLibrary(filename) -if libc is not None: -have_unshare = hasattr(libc, "unshare") +(libc, _) = load_libc() +if libc is not None: +have_unshare = hasattr(libc, "unshare") if not have_unshare: # unshare() may not be supported by libc @@ -1212,11 +1210,7 @@ class _unshare_validator: """ # This ctypes library lookup caches the result for use in the # subprocess when the multiprocessing start method is fork. -filename = find_library("c") -if filename is None: -return errno.ENOTSUP - -libc = LoadLibrary(filename) +(libc, filename) = load_libc() if libc is None: return errno.ENOTSUP diff --git a/lib/portage/util/_ctypes.py b/lib/portage/util/_ctypes.py index e6d1e327cb..04e965ba92 100644 --- a/lib/portage/util/_ctypes.py +++ b/lib/portage/util/_ctypes.py @@ -48,3
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 0855821572f32e81b031a250f7491f34a2fd4078 Author: Zac Medico gentoo org> AuthorDate: Sat Feb 24 23:29:29 2024 + Commit: Sam James gentoo org> CommitDate: Sun Feb 25 08:24:55 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=08558215 dbapi: Fix TypeError when passing Exception to warnings.warn If an Exception is passed as a message to warnings.warn then it triggers this TypeError: if metadata_updates: try: aux_update(cpv, metadata_updates) except (InvalidBinaryPackageFormat, CorruptionKeyError) as e: > warnings.warn(e) E TypeError: expected string or bytes-like object, got 'CorruptionKeyError' Fixes: fb1d0a22f657 ("dbapi: KeyError tolerance during package moves") Bug: https://bugs.gentoo.org/922935 Signed-off-by: Zac Medico gentoo.org> Closes: https://github.com/gentoo/portage/pull/1282 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/__init__.py | 11 +++ 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py index 6f95b93a21..9105227c77 100644 --- a/lib/portage/dbapi/__init__.py +++ b/lib/portage/dbapi/__init__.py @@ -1,11 +1,12 @@ -# Copyright 1998-2023 Gentoo Authors +# Copyright 1998-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["dbapi"] import functools +import logging import re -import warnings +import sys from typing import Any, Dict, List, Optional, Tuple from collections.abc import Sequence @@ -429,7 +430,9 @@ class dbapi: try: aux_update(cpv, metadata_updates) except (InvalidBinaryPackageFormat, CorruptionKeyError) as e: -warnings.warn(e) +logging.warning( +f"{e.__class__.__name__}: {e}", exc_info=sys.exc_info() +) if onUpdate: onUpdate(maxval, i + 1) if onProgress: @@ -477,6 +480,6 @@ class dbapi: try: self.aux_update(mycpv, mydata) except CorruptionKeyError as e: -warnings.warn(e) +logging.warning(f"{e.__class__.__name__}: {e}", exc_info=sys.exc_info()) continue return moves
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: adc0052b2c022851458f788868e6b194ed1cfe9f Author: Sam James gentoo org> AuthorDate: Tue Jan 16 07:51:47 2024 + Commit: Sam James gentoo org> CommitDate: Tue Jan 16 07:52:11 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=adc0052b bintree: add missing newlines to signed binpkg update notice Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/bintree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index ee574f435f..f4251b47d6 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -306,7 +306,7 @@ class bindbapi(fakedbapi): colorize( "WARN", f"Binpkg update ignored for signed package: {binpkg_path}, " -"the file will be removed.", +"the file will be removed.\n", ) ) self.bintree.remove(cpv) @@ -734,7 +734,7 @@ class binarytree: writemsg( colorize( "WARN", -f"Binpkg update ignored for signed package: {binpkg_path}", +f"Binpkg update ignored for signed package: {binpkg_path}\n", ) ) continue
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 68f4ea8a90d8759a1aa859d9188017e21797bdd0 Author: Zac Medico gentoo org> AuthorDate: Tue Jan 16 01:08:36 2024 + Commit: Zac Medico gentoo org> CommitDate: Tue Jan 16 05:25:20 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=68f4ea8a Handle SignatureException during package moves Ignore package moves for packages that raise SignatureException, just as they are ignored for packages that have a valid signature. Bug: https://bugs.gentoo.org/922142 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 16 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 2342d571b9..ee574f435f 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -296,8 +296,12 @@ class bindbapi(fakedbapi): encoding_key = True elif binpkg_format == "gpkg": mybinpkg = portage.gpkg.gpkg(self.settings, cpv_str, binpkg_path) -mydata = mybinpkg.get_metadata() -if mybinpkg.signature_exist: +try: +mydata = mybinpkg.get_metadata() +signature_exist = mybinpkg.signature_exist +except SignatureException: +signature_exist = True +if signature_exist: writemsg( colorize( "WARN", @@ -721,8 +725,12 @@ class binarytree: decode_metadata_name = False elif binpkg_format == "gpkg": mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, binpkg_path) -mydata = mybinpkg.get_metadata() -if mybinpkg.signature_exist: +try: +mydata = mybinpkg.get_metadata() +signature_exist = mybinpkg.signature_exist +except SignatureException: +signature_exist = True +if signature_exist: writemsg( colorize( "WARN",
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: c9ed8136f9f0c56dc7b72d400eaa0acbd338778f Author: Konstantin Tokarev yandex ru> AuthorDate: Fri Jan 12 17:47:45 2024 + Commit: Sam James gentoo org> CommitDate: Tue Jan 16 05:16:08 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c9ed8136 vartree: _needs_move() should replace file if filecmp fails for whatever reason When trying to update I've got "OSError: [Errno 5] Input/output error" exception traceback from portage originating from _needs_move(). It turned out that 3 files in my root filesystem were corrupted, which caused filecmp to fail. This patch makes portage replace original file in this case. Bug: https://bugs.gentoo.org/722270 Signed-off-by: Konstantin Tokarev yandex.ru> Closes: https://github.com/gentoo/portage/pull/1233 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 14 +- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 5d39ca1965..a00c731c57 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -6283,7 +6283,19 @@ class dblink: if not _cmpxattr(src_bytes, dest_bytes, exclude=excluded_xattrs): return True -return not filecmp.cmp(src_bytes, dest_bytes, shallow=False) +try: +files_equal = filecmp.cmp(src_bytes, dest_bytes, shallow=False) +except Exception as e: +writemsg( +_( +"Exception '%s' happened when comparing files %s and %s, will replace the latter\n" +) +% (e, mysrc, mydest), +noiselevel=-1, +) +return True + +return not files_equal def merge(
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 58b094bc79e999e44a5b108e2b7273c164aa906e Author: Alfred Wingate protonmail com> AuthorDate: Fri Jan 5 19:30:38 2024 + Commit: Sam James gentoo org> CommitDate: Tue Jan 16 05:16:09 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=58b094bc bintree: use urllib provided attributes for hostname, user and password In addition to simplifying it also solves an issue with parsing raw ipv6 addresses. Bug: https://bugs.gentoo.org/921400 Signed-off-by: Alfred Wingate protonmail.com> Closes: https://github.com/gentoo/portage/pull/1230 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/bintree.py | 18 -- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 5fc4716980..2342d571b9 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1344,23 +1344,13 @@ class binarytree: for repo in reversed(list(self._binrepos_conf.values())): base_url = repo.sync_uri parsed_url = urlparse(base_url) -host = parsed_url.netloc +host = parsed_url.hostname or "" port = parsed_url.port -user = None -passwd = None -user_passwd = "" +user = parsed_url.username +passwd = parsed_url.password +user_passwd = user + "@" if user else "" gpkg_only_warned = False -if "@" in host: -user, host = host.split("@", 1) -user_passwd = user + "@" -if ":" in user: -user, passwd = user.split(":", 1) - -if port is not None: -port_str = f":{port}" -if host.endswith(port_str): -host = host[: -len(port_str)] pkgindex_file = os.path.join( self.settings["EROOT"], CACHE_PATH,
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 6c915f3ded10c23127c33b771453d54e3f0d4915 Author: Zac Medico gentoo org> AuthorDate: Wed Jan 3 04:26:36 2024 + Commit: Zac Medico gentoo org> CommitDate: Wed Jan 3 05:56:59 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c915f3d binarytree: Avoid unnecessary build id incrementation When allocating a local file name for a remote package, prefer re-using the existing build id if the file name is unused locally, in order to avoid unnecessary build id incrementation triggered when the _max_build_id method counts remote build ids. Use the same file name reservation via open with O_EXCL which has been used during build_id incrementation since fa901a6510c. Bug: https://bugs.gentoo.org/921208 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 39 --- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index d352b6fc0e..5fc4716980 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2023 Gentoo Authors +# Copyright 1998-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["bindbapi", "binarytree"] @@ -2272,16 +2272,6 @@ class binarytree: raise InvalidBinaryPackageFormat(binpkg_format) def _allocate_filename_multi(self, cpv, remote_binpkg_format=None): -# First, get the max build_id found when _populate was -# called. -max_build_id = self._max_build_id(cpv) - -# A new package may have been added concurrently since the -# last _populate call, so use increment build_id until -# we locate an unused id. -pf = catsplit(cpv)[1] -build_id = max_build_id + 1 - if remote_binpkg_format is None: try: binpkg_format = get_binpkg_format(cpv._metadata["PATH"]) @@ -2299,6 +2289,33 @@ class binarytree: else: raise InvalidBinaryPackageFormat(binpkg_format) +# If the preferred path is available then return +# that. This prevents unnecessary build_id incrementation +# triggered when the _max_build_id method counts remote +# build ids. +pf = catsplit(cpv)[1] +if getattr(cpv, "build_id", False): +preferred_path = f"{os.path.join(self.pkgdir, cpv.cp, pf)}-{cpv.build_id}.{binpkg_suffix}" +if not os.path.exists(preferred_path): +try: +# Avoid races +ensure_dirs(os.path.dirname(preferred_path)) +with open(preferred_path, "x") as f: +pass +except FileExistsError: +pass +else: +return (preferred_path, cpv.build_id) + +# First, get the max build_id found when _populate was +# called. +max_build_id = self._max_build_id(cpv) + +# A new package may have been added concurrently since the +# last _populate call, so use increment build_id until +# we locate an unused id. +build_id = max_build_id + 1 + while True: filename = ( f"{os.path.join(self.pkgdir, cpv.cp, pf)}-{build_id}.{binpkg_suffix}"
[gentoo-commits] proj/portage:master commit in: /, lib/portage/dbapi/
commit: b071a966357a29e8450688a28ca918166b9e4eb0 Author: Sam James gentoo org> AuthorDate: Tue Jan 2 04:00:02 2024 + Commit: Sam James gentoo org> CommitDate: Tue Jan 2 04:09:56 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b071a966 bintree: don't call trust helper unless bindb is writable Followup to 6ae45739e208b7a9d59e0b6056be72a5791aae04. My qualm there wrt writable was whether or not doing something which mutated state (and therefore possibly the package list) would be confusing but that doesn't make much sense for a few reasons. Anyway, change the test to be not just for no-pretend, but also whether the bindb is writable too, as pretend is already a proxy for whether we may not have privileges (I can imagine someone possibly having bindb privileges but not /etc/portage/gnupg, so better to just head this off entirely). Bug: https://bugs.gentoo.org/915842 Bug: https://bugs.gentoo.org/920180 Signed-off-by: Sam James gentoo.org> NEWS | 3 ++- lib/portage/dbapi/bintree.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 038a4dbf8c..2ee334f20c 100644 --- a/NEWS +++ b/NEWS @@ -9,7 +9,8 @@ Release notes take the form of the following optional categories: portage-3.0.61 (UNRELEASED) -- -TODO +Bug fixes: +* bintree: Don't call trust helper unless bindb is writable (bug #915842, bug #920180). portage-3.0.60 (2024-01-02) -- diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index a139e37659..d352b6fc0e 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1334,7 +1334,7 @@ class binarytree: # when binpackages are involved, not only when we refuse unsigned # ones. (If the keys have expired we end up refusing signed but # technically invalid packages...) -if not pretend: +if not pretend and self.dbapi.writable: self._run_trust_helper() gpkg_only = True else:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/_async/, lib/portage/tests/emerge/, ...
commit: 1db44d18578a7aee58449cb97e1991cb06c915c3 Author: Zac Medico gentoo org> AuthorDate: Tue Dec 26 06:42:28 2023 + Commit: Zac Medico gentoo org> CommitDate: Tue Dec 26 06:42:28 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1db44d18 bintree: support file scheme for binhost src-uri Add local file scheme support for binhost src-uri in bintree and BinpkgFetcher. Make BinpkgFetcher use a coroutine to avoid using callbacks. Add AsyncTaskFuture isAlive method that BinpkgFetcher can use to check FileCopier state. Add test command to test emerge -f --getbinpkgonly with local file scheme in binrepos.conf. Also, fix the getbinpkgonly_fetchonly test command so that it does not rename binhost_dir to pkgdir. It seems like it should not do this, and it caused the getbinpkgonly_file_uri command to fail. Bug: https://bugs.gentoo.org/920537 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/BinpkgFetcher.py | 117 +++-- lib/portage/dbapi/bintree.py | 19 +++-- lib/portage/tests/emerge/conftest.py | 21 +- lib/portage/util/_async/AsyncTaskFuture.py | 8 +- 4 files changed, 111 insertions(+), 54 deletions(-) diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index 10f9b6e427..01b2bae637 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -1,8 +1,6 @@ -# Copyright 1999-2020 Gentoo Authors +# Copyright 1999-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import functools - from _emerge.AsynchronousLock import AsynchronousLock from _emerge.CompositeTask import CompositeTask from _emerge.SpawnProcess import SpawnProcess @@ -14,6 +12,7 @@ from portage import os from portage.binpkg import get_binpkg_format from portage.exception import FileNotFound from portage.util._async.AsyncTaskFuture import AsyncTaskFuture +from portage.util._async.FileCopier import FileCopier from portage.util._pty import _create_pty_or_pipe @@ -40,6 +39,22 @@ class BinpkgFetcher(CompositeTask): self.pkg_path = self.pkg_allocated_path + ".partial" def _start(self): +self._start_task( +AsyncTaskFuture(future=self._main(), scheduler=self.scheduler), +self._main_exit, +) + +async def _main(self) -> int: +""" +Main coroutine which saves the binary package to self.pkg_path +and returns the exit status of the fetcher or copier. + +@rtype: int +@return: Exit status of fetcher or copier. +""" +pkg = self.pkg +bintree = pkg.root_config.trees["bintree"] + fetcher = _BinpkgFetcherProcess( background=self.background, logfile=self.logfile, @@ -52,47 +67,67 @@ class BinpkgFetcher(CompositeTask): if not self.pretend: portage.util.ensure_dirs(os.path.dirname(self.pkg_path)) if "distlocks" in self.pkg.root_config.settings.features: -self._start_task( -AsyncTaskFuture(future=fetcher.async_lock()), -functools.partial(self._start_locked, fetcher), -) -return - -self._start_task(fetcher, self._fetcher_exit) - -def _start_locked(self, fetcher, lock_task): -self._assert_current(lock_task) -if lock_task.cancelled: -self._default_final_exit(lock_task) -return - -lock_task.future.result() -self._start_task(fetcher, self._fetcher_exit) - -def _fetcher_exit(self, fetcher): -self._assert_current(fetcher) -if not self.pretend and fetcher.returncode == os.EX_OK: -fetcher.sync_timestamp() -if fetcher.locked: -self._start_task( -AsyncTaskFuture(future=fetcher.async_unlock()), -functools.partial(self._fetcher_exit_unlocked, fetcher), -) -else: -self._fetcher_exit_unlocked(fetcher) +await fetcher.async_lock() + +try: +if bintree._remote_has_index: +remote_metadata = bintree._remotepkgs[ +bintree.dbapi._instance_key(pkg.cpv) +] +rel_uri = remote_metadata.get("PATH") +if not rel_uri: +# Assume that the remote index is out of date. No path should +# never happen in new portage versions. +rel_uri = pkg.cpv + ".tbz2" +remote_base_uri = remote_metadata["BASE_URI"] +uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/") +else: +raise FileNotFound("Binary packages index not found") -def _fetcher_exit_unlocked(self, fetcher, unlock_task=None): -if unlock_task is not None: -self._assert_current(unlock_task) -if unlock_task.cancelled: -
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: cc372a52ee2aba153c0d7c2290306b7f623e8b7c Author: Zac Medico gentoo org> AuthorDate: Sat Dec 9 23:58:07 2023 + Commit: Zac Medico gentoo org> CommitDate: Sun Dec 10 01:27:43 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cc372a52 bindbapi: add SIZE to _pkg_str_aux_keys The _pkg_str_aux_keys are used inside dbapi.update_ents, and need to contain SIZE in order for the fakedbapi _instance_key_multi_instance method to operate correctly. Incorrect operation of _instance_key_multi_instance could prevent binarytree.inject from removing an old instance from its internal state. It could also trigger a KeyError in bindbapi.aux_update as in bug 918597, since it could cause binarytree.getname to return a non-existent path. It could also cause binarytree.getname to return an existing but incorrect path, which might trigger an InvalidBinaryPackageFormat exception as in bug 906675. Bug: https://bugs.gentoo.org/906675 Bug: https://bugs.gentoo.org/918597 Bug: https://bugs.gentoo.org/919668 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index a6e1f9773d..9c9ac66334 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -84,9 +84,12 @@ class bindbapi(fakedbapi): _known_keys = frozenset( list(fakedbapi._known_keys) + ["CHOST", "repository", "USE"] ) +# Must include keys used to create _pkg_str attributes used in +# the fakedbapi _instance_key_multi_instance method. _pkg_str_aux_keys = fakedbapi._pkg_str_aux_keys + ( "BUILD_ID", "BUILD_TIME", +"SIZE", "_mtime_", )
[gentoo-commits] proj/portage:master commit in: /, lib/portage/dbapi/, lib/portage/tests/dbapi/, lib/_emerge/, ...
commit: 6ae45739e208b7a9d59e0b6056be72a5791aae04 Author: Sam James gentoo org> AuthorDate: Mon Nov 6 17:07:10 2023 + Commit: Sam James gentoo org> CommitDate: Mon Nov 6 17:25:04 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6ae45739 bintree: don't call trust helper with --pretend Trust helpers are likely to need privileges and it's a bit too far for pretend there, I think. People can run it manually if they want it done for them anyway. We could check writable instead but I'd like to get a fix in for the regression first. Bug: https://bugs.gentoo.org/915842 Signed-off-by: Sam James gentoo.org> NEWS | 3 +++ lib/_emerge/actions.py | 3 +++ lib/portage/dbapi/bintree.py | 14 +++--- lib/portage/tests/dbapi/test_bintree.py | 6 -- lib/portage/tests/emerge/test_actions.py | 4 +++- 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index 89d9335275..654b3175f0 100644 --- a/NEWS +++ b/NEWS @@ -17,6 +17,9 @@ Bug fixes: * Avoid crash with blockers in depgraph for binpkg-respect-use notice (bug #916336). +* Don't call trust helper (e.g. getuto) with --getbinpkg --pretend as we may + lack privileges to do anything (bug #915842). + portage-3.0.54 (2023-10-25) -- diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index 323fde3767..07d477a046 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -174,6 +174,7 @@ def action_build( kwargs["add_repos"] = (quickpkg_vardb,) try: +kwargs["pretend"] = "--pretend" in emerge_config.opts emerge_config.target_config.trees["bintree"].populate( getbinpkgs="--getbinpkg" in emerge_config.opts, **kwargs ) @@ -3472,6 +3473,8 @@ def run_action(emerge_config): emerge_config.running_config.trees["vartree"].dbapi, ) +kwargs["pretend"] = "--pretend" in emerge_config.opts + try: mytrees["bintree"].populate( getbinpkgs="--getbinpkg" in emerge_config.opts, diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 0ecfdc25d0..6446fde95a 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -831,6 +831,7 @@ class binarytree: add_repos=(), force_reindex=False, invalid_errors=True, +pretend=False, ): """ Populates the binarytree with package metadata. @@ -844,6 +845,10 @@ class binarytree: @type add_repos: sequence """ +# TODO: Should we return here if we're --pretend? On the one hand, +# people might not want --pretend to affect state. On the other hand, +# it makes --pretend pretty useless with --getbinpkg as your index will +# be stale. if self._populating: return @@ -898,7 +903,9 @@ class binarytree: noiselevel=-1, ) else: -self._populate_remote(getbinpkg_refresh=getbinpkg_refresh) +self._populate_remote( +getbinpkg_refresh=getbinpkg_refresh, pretend=pretend +) finally: self._populating = False @@ -1290,7 +1297,7 @@ class binarytree: return ret.check_returncode() -def _populate_remote(self, getbinpkg_refresh=True): +def _populate_remote(self, getbinpkg_refresh=True, pretend=False): self._remote_has_index = False self._remotepkgs = {} @@ -1299,7 +1306,8 @@ class binarytree: # when binpackages are involved, not only when we refuse unsigned # ones. (If the keys have expired we end up refusing signed but # technically invalid packages...) -self._run_trust_helper() +if not pretend: +self._run_trust_helper() gpkg_only = True else: gpkg_only = False diff --git a/lib/portage/tests/dbapi/test_bintree.py b/lib/portage/tests/dbapi/test_bintree.py index 5a6ee5b142..0aa411ad97 100644 --- a/lib/portage/tests/dbapi/test_bintree.py +++ b/lib/portage/tests/dbapi/test_bintree.py @@ -120,7 +120,9 @@ class BinarytreeTestCase(TestCase): settings.__getitem__.return_value = "/some/path" bt = binarytree(pkgdir=os.getenv("TMPDIR", "/tmp"), settings=settings) bt.populate(getbinpkgs=True, getbinpkg_refresh=refresh) -ppopulate_remote.assert_called_once_with(getbinpkg_refresh=refresh) +ppopulate_remote.assert_called_once_with( +getbinpkg_refresh=refresh, pretend=False +) @patch("portage.dbapi.bintree.writemsg") @patch("portage.dbapi.bintree.BinRepoConfigLoader") @@ -161,4 +163,4 @@ class BinarytreeTestCase(TestCase): settings.__getitem__.
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 7a43de7a8ce8dc6836d998cb87a5135ba8f0bc2c Author: Zac Medico gentoo org> AuthorDate: Tue Oct 24 18:07:53 2023 + Commit: Zac Medico gentoo org> CommitDate: Tue Oct 24 18:08:23 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a43de7a dbapi: Make _iuse_implicit_cnstr picklable for spawn compat Bug: https://bugs.gentoo.org/916235 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/__init__.py | 15 +++ 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py index 233da8d698..dd697db109 100644 --- a/lib/portage/dbapi/__init__.py +++ b/lib/portage/dbapi/__init__.py @@ -1,8 +1,9 @@ -# Copyright 1998-2020 Gentoo Authors +# Copyright 1998-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["dbapi"] +import functools import re import warnings from typing import Any, Dict, List, Optional, Tuple @@ -223,6 +224,10 @@ class dbapi: yield cpv +@staticmethod +def _iuse_implicit_built(iuse_implicit_match, use, flag): +return iuse_implicit_match(flag) or flag in use + def _iuse_implicit_cnstr(self, pkg, metadata): """ Construct a callable that checks if a given USE flag should @@ -257,9 +262,11 @@ class dbapi: # This behavior is only used for EAPIs that support IUSE_EFFECTIVE, # since built USE settings for earlier EAPIs may contain a large # number of irrelevant flags. -prof_iuse = iuse_implicit_match -enabled = frozenset(metadata["USE"].split()).__contains__ -iuse_implicit_match = lambda flag: prof_iuse(flag) or enabled(flag) +iuse_implicit_match = functools.partial( +self._iuse_implicit_built, +iuse_implicit_match, +frozenset(metadata["USE"].split()), +) return iuse_implicit_match
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: c7768ae4820201913ba28d5fad441c1acc4659c0 Author: Zac Medico gentoo org> AuthorDate: Mon Oct 23 04:14:05 2023 + Commit: Zac Medico gentoo org> CommitDate: Mon Oct 23 04:41:10 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c7768ae4 bindbapi: Fix pickling for MergeProcess Both the _aux_cache_slot_dict_cache and _instance_key attributes are unpicklable, so fix __getstate__ to exclude them both from the state. Override __setstate__ to initialize _instance_key appropriately. These changes prevent the following error: AttributeError: Can't pickle local object 'slot_dict_class..LocalSlotDict' Fixes: b9a85ff987ea ("MergeProcess: Support QueryCommand with spawn start method") Bug: https://bugs.gentoo.org/916106 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 21 + 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 3f1bf9d5d2..0ecfdc25d0 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -139,14 +139,19 @@ class bindbapi(fakedbapi): return self._aux_cache_slot_dict_cache def __getstate__(self): -# This attribute is not picklable, but it automatically -# regenerates after unpickling when set to None. -_aux_cache_slot_dict = self._aux_cache_slot_dict_cache -self._aux_cache_slot_dict_cache = None -try: -return super().__getstate__() -finally: -self._aux_cache_slot_dict_cache = _aux_cache_slot_dict +state = self.__dict__.copy() +# These attributes are not picklable, so they are automatically +# regenerated after unpickling. +state["_aux_cache_slot_dict_cache"] = None +state["_instance_key"] = None +return state + +def __setstate__(self, state): +self.__dict__.update(state) +if self._multi_instance: +self._instance_key = self._instance_key_multi_instance +else: +self._instance_key = self._instance_key_cpv @property def writable(self):
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: b9a85ff987ea677e350fb70e2c3355197ee6c39b Author: Zac Medico gentoo org> AuthorDate: Sun Oct 22 08:08:02 2023 + Commit: Zac Medico gentoo org> CommitDate: Sun Oct 22 21:29:49 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9a85ff9 MergeProcess: Support QueryCommand with spawn start method Since QueryCommand._db does not propagate to the child process in MergeProcess with the multiprocessing spawn start method, handle it by passing QueryCommand._db as an explicit parameter. Since self.mydbapi can be a bindbapi instance, override the bindbapi __getstate__ method so that it is picklable (omit the unpicklable _aux_cache_slot_dict attribute and regenerate it after unpickling). Bug: https://bugs.gentoo.org/916106 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/_MergeProcess.py | 19 --- lib/portage/dbapi/bintree.py | 20 ++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index cf8e6513d1..dd5ad71cf8 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -9,6 +9,7 @@ import platform import fcntl import portage from portage import os, _unicode_decode +from portage.package.ebuild._ipc.QueryCommand import QueryCommand from portage.util._ctypes import find_library import portage.elog.messages from portage.util._async.ForkProcess import ForkProcess @@ -180,6 +181,15 @@ class MergeProcess(ForkProcess): self._dblink = mylink self._elog_reader_fd = elog_reader_fd +# Since the entire QueryCommand._db is not required, only pass +# in tree types that QueryCommand specifically requires. +child_db = {} +parent_db = portage.db if QueryCommand._db is None else QueryCommand._db +for root in parent_db: +child_db[root] = {} +for tree_type in ("vartree", "porttree"): +child_db[root][tree_type] = parent_db[root][tree_type] + self.target = functools.partial( self._target, self._counter, @@ -192,6 +202,7 @@ class MergeProcess(ForkProcess): self.settings, self.unmerge, self.vartree.dbapi, +child_db, ) pids = super()._spawn(args, fd_pipes, **kwargs) @@ -223,10 +234,12 @@ class MergeProcess(ForkProcess): settings, unmerge, vardb, +db, ): -""" -TODO: Make all arguments picklable for the multiprocessing spawn start method. -""" +if QueryCommand._db is None: +# Initialize QueryCommand._db for AbstractEbuildProcess/EbuildIpcDaemon +# when not using the multiprocessing fork start method. +QueryCommand._db = db portage.output.havecolor = not no_color(settings) # Avoid wastful updates of the vdb cache. vardb._flush_cache_enabled = False diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 421863f3e8..3f1bf9d5d2 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2021 Gentoo Authors +# Copyright 1998-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["bindbapi", "binarytree"] @@ -129,8 +129,24 @@ class bindbapi(fakedbapi): "USE", "_mtime_", } -self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys) self._aux_cache = {} +self._aux_cache_slot_dict_cache = None + +@property +def _aux_cache_slot_dict(self): +if self._aux_cache_slot_dict_cache is None: +self._aux_cache_slot_dict_cache = slot_dict_class(self._aux_cache_keys) +return self._aux_cache_slot_dict_cache + +def __getstate__(self): +# This attribute is not picklable, but it automatically +# regenerates after unpickling when set to None. +_aux_cache_slot_dict = self._aux_cache_slot_dict_cache +self._aux_cache_slot_dict_cache = None +try: +return super().__getstate__() +finally: +self._aux_cache_slot_dict_cache = _aux_cache_slot_dict @property def writable(self):
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 0b9fbe389db3c3b9e625e1f5f526b3a921d2e0ce Author: Zac Medico gentoo org> AuthorDate: Sun Oct 22 19:57:16 2023 + Commit: Zac Medico gentoo org> CommitDate: Sun Oct 22 19:58:20 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0b9fbe38 vardbapi.unpack_contents: Use multiprocessing.Pipe for spawn compat Bug: https://bugs.gentoo.org/916112 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 28 +++- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index a80d3bc0e3..5638935606 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -101,6 +101,7 @@ import grp import io from itertools import chain import logging +import multiprocessing import os as _os import operator import pickle @@ -1094,19 +1095,18 @@ class vardbapi(dbapi): ) if binpkg_format == "xpak": tar_cmd = ("tar", "-x", "--xattrs", "--xattrs-include=*", "-C", dest_dir) -pr, pw = os.pipe() +pr, pw = multiprocessing.Pipe(duplex=False) proc = await asyncio.create_subprocess_exec(*tar_cmd, stdin=pr) -os.close(pr) -with os.fdopen(pw, "wb", 0) as pw_file: -excluded_config_files = await loop.run_in_executor( -ForkExecutor(loop=loop), -functools.partial( -self._dblink(cpv).quickpkg, -pw_file, -include_config=opts.include_config == "y", - include_unmodified_config=opts.include_unmodified_config == "y", -), -) +pr.close() +excluded_config_files = await loop.run_in_executor( +ForkExecutor(loop=loop), +functools.partial( +self._dblink(cpv).quickpkg, +pw, +include_config=opts.include_config == "y", +include_unmodified_config=opts.include_unmodified_config == "y", +), +) await proc.wait() if proc.returncode != os.EX_OK: raise PortageException(f"command failed: {tar_cmd}") @@ -2161,7 +2161,9 @@ class dblink: # The tarfile module will write pax headers holding the # xattrs only if PAX_FORMAT is specified here. with tarfile.open( -fileobj=output_file, +fileobj=output_file +if hasattr(output_file, "write") +else open(output_file.fileno(), mode="wb", closefd=False), mode="w|", format=tarfile.PAX_FORMAT if xattrs else tarfile.DEFAULT_FORMAT, ) as tar:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: c41d109b121feb0bf6199b4082a35799fdc547b9 Author: Zac Medico gentoo org> AuthorDate: Sun Oct 22 08:28:39 2023 + Commit: Zac Medico gentoo org> CommitDate: Sun Oct 22 08:30:50 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c41d109b vardbapi.unpack_metadata: Make compatible with spawn start method This solves the following error with the multiprocessing spawn start method: AttributeError: Can't pickle local object 'vardbapi.unpack_metadata..async_copy' Bug: https://bugs.gentoo.org/916108 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 18 ++ 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 88fc525771..a80d3bc0e3 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -1003,6 +1003,13 @@ class vardbapi(dbapi): pass self._bump_mtime(cpv) +@staticmethod +def _async_copy(dbdir, dest_dir): +for parent, dirs, files in os.walk(dbdir, onerror=_raise_exc): +for key in files: +shutil.copy(os.path.join(parent, key), os.path.join(dest_dir, key)) +break + async def unpack_metadata(self, pkg, dest_dir, loop=None): """ Unpack package metadata to a directory. This method is a coroutine. @@ -1018,14 +1025,9 @@ class vardbapi(dbapi): else: cpv = pkg.mycpv dbdir = self.getpath(cpv) - -def async_copy(): -for parent, dirs, files in os.walk(dbdir, onerror=_raise_exc): -for key in files: -shutil.copy(os.path.join(parent, key), os.path.join(dest_dir, key)) -break - -await loop.run_in_executor(ForkExecutor(loop=loop), async_copy) +await loop.run_in_executor( +ForkExecutor(loop=loop), self._async_copy, dbdir, dest_dir +) async def unpack_contents( self,
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: d92d69a52b2b127a0934656163f7075015ef7c85 Author: Zac Medico gentoo org> AuthorDate: Tue Oct 17 19:05:07 2023 + Commit: Zac Medico gentoo org> CommitDate: Tue Oct 17 19:14:54 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d92d69a5 MergeProcess: Use multiprocessing.Pipe to decouple fd_pipes Use multiprocessing.Pipe to decouple from the fd_pipes implementation since that currently only works for the multiprocessing "fork" start method. Bug: https://bugs.gentoo.org/915903 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/_MergeProcess.py | 25 ++--- lib/portage/dbapi/vartree.py | 2 +- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index c1085270da..cf8e6513d1 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -96,7 +96,7 @@ class MergeProcess(ForkProcess): self._locked_vdb = False def _elog_output_handler(self): -output = self._read_buf(self._elog_reader_fd) +output = self._read_buf(self._elog_reader_fd.fileno()) if output: lines = _unicode_decode(output).split("\n") if len(lines) == 1: @@ -112,8 +112,8 @@ class MergeProcess(ForkProcess): reporter(msg, phase=phase, key=key, out=out) elif output is not None: # EIO/POLLHUP -self.scheduler.remove_reader(self._elog_reader_fd) -os.close(self._elog_reader_fd) +self.scheduler.remove_reader(self._elog_reader_fd.fileno()) +self._elog_reader_fd.close() self._elog_reader_fd = None return False @@ -136,16 +136,15 @@ class MergeProcess(ForkProcess): post-fork actions. """ -elog_reader_fd, elog_writer_fd = os.pipe() +elog_reader_fd, elog_writer_fd = multiprocessing.Pipe(duplex=False) fcntl.fcntl( -elog_reader_fd, +elog_reader_fd.fileno(), fcntl.F_SETFL, -fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK, +fcntl.fcntl(elog_reader_fd.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK, ) mtime_reader, mtime_writer = multiprocessing.Pipe(duplex=False) -fd_pipes[mtime_writer.fileno()] = mtime_writer.fileno() self.scheduler.add_reader(mtime_reader.fileno(), self._mtime_handler) self._mtime_reader = mtime_reader @@ -166,8 +165,7 @@ class MergeProcess(ForkProcess): pipe=elog_writer_fd, mtime_pipe=mtime_writer, ) -fd_pipes[elog_writer_fd] = elog_writer_fd -self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler) +self.scheduler.add_reader(elog_reader_fd.fileno(), self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an @@ -185,7 +183,6 @@ class MergeProcess(ForkProcess): self.target = functools.partial( self._target, self._counter, -self._elog_reader_fd, self._dblink, self.infloc, self.mydbapi, @@ -198,7 +195,7 @@ class MergeProcess(ForkProcess): ) pids = super()._spawn(args, fd_pipes, **kwargs) -os.close(elog_writer_fd) +elog_writer_fd.close() mtime_writer.close() self._buf = "" self._elog_keys = set() @@ -217,7 +214,6 @@ class MergeProcess(ForkProcess): @staticmethod def _target( counter, -elog_reader_fd, mylink, infloc, mydbapi, @@ -231,7 +227,6 @@ class MergeProcess(ForkProcess): """ TODO: Make all arguments picklable for the multiprocessing spawn start method. """ -os.close(elog_reader_fd) portage.output.havecolor = not no_color(settings) # Avoid wastful updates of the vdb cache. vardb._flush_cache_enabled = False @@ -301,8 +296,8 @@ class MergeProcess(ForkProcess): self._unlock_vdb() if self._elog_reader_fd is not None: -self.scheduler.remove_reader(self._elog_reader_fd) -os.close(self._elog_reader_fd) +self.scheduler.remove_reader(self._elog_reader_fd.fileno()) +self._elog_reader_fd.close() self._elog_reader_fd = None if self._elog_keys is not None: for key in self._elog_keys: diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 835cbb8092..88fc525771 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -4198,7 +4198,7 @@ class dblink: if str_buffer: str_buffer = _unicode_encode("".join(str_buffer)) while str_buffer: -str_buffer = str_buffer[os.write(self._pipe
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 3e38ae92bdd5b057352a2bcb044fb587b15b25f3 Author: Zac Medico gentoo org> AuthorDate: Sun Oct 15 19:21:44 2023 + Commit: Zac Medico gentoo org> CommitDate: Sun Oct 15 21:38:08 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3e38ae92 MergeProcess: Eliminate target arguments that reference self This improves compatibility with the multiprocessing spawn start method, by eliminating this error: AttributeError: Can't pickle local object 'MergeProcess._start..' Bug: https://bugs.gentoo.org/914876 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/_MergeProcess.py | 40 +++--- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index 012435dce3..c1085270da 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -74,20 +74,6 @@ class MergeProcess(ForkProcess): self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD") -self.target = functools.partial( -self._target, -lambda: self._counter, -lambda: self._elog_reader_fd, -lambda: self._dblink, -self.infloc, -self.mydbapi, -self.myebuild, -self.pkgloc, -self.prev_mtimes, -self.settings, -self.unmerge, -self.vartree.dbapi, -) super()._start() def _lock_vdb(self): @@ -195,6 +181,22 @@ class MergeProcess(ForkProcess): self._dblink = mylink self._elog_reader_fd = elog_reader_fd + +self.target = functools.partial( +self._target, +self._counter, +self._elog_reader_fd, +self._dblink, +self.infloc, +self.mydbapi, +self.myebuild, +self.pkgloc, +self.prev_mtimes, +self.settings, +self.unmerge, +self.vartree.dbapi, +) + pids = super()._spawn(args, fd_pipes, **kwargs) os.close(elog_writer_fd) mtime_writer.close() @@ -214,9 +216,9 @@ class MergeProcess(ForkProcess): @staticmethod def _target( -get_counter, -get_elog_reader_fd, -get_mylink, +counter, +elog_reader_fd, +mylink, infloc, mydbapi, myebuild, @@ -229,9 +231,7 @@ class MergeProcess(ForkProcess): """ TODO: Make all arguments picklable for the multiprocessing spawn start method. """ -os.close(get_elog_reader_fd()) -counter = get_counter() -mylink = get_mylink() +os.close(elog_reader_fd) portage.output.havecolor = not no_color(settings) # Avoid wastful updates of the vdb cache. vardb._flush_cache_enabled = False
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: fc70635aecbf80d3aeeb9f4e56d396040da09279 Author: Siddhanth Rathod gmail com> AuthorDate: Sun Oct 8 09:25:15 2023 + Commit: Zac Medico gentoo org> CommitDate: Sun Oct 8 19:47:56 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fc70635a Fix: move msg array such that it gets assigned only when needed Signed-off-by: Siddhanth Rathod gmail.com> Closes: https://github.com/gentoo/portage/pull/1124 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 7a4166c120..421863f3e8 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1092,10 +1092,10 @@ class binarytree: missing_keys.append("PF") if not slot: missing_keys.append("SLOT") -msg = [] -if missing_keys: -missing_keys.sort() -if invalid_errors: +if invalid_errors: +msg = [] +if missing_keys: +missing_keys.sort() msg.append( _("Missing metadata key(s): %s.") % ", ".join(missing_keys)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/dbapi/
commit: c53d046c9629d5c3a7841aee4e92ae38c0691e69 Author: Siddhanth Rathod gmail com> AuthorDate: Thu Oct 5 13:22:52 2023 + Commit: Sam James gentoo org> CommitDate: Sun Oct 8 03:29:46 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c53d046c dbapi: bintree: introduce invalid_errors Introduce 'invalid_errors' var to enable suppression of invalid binary error, for use by gentoolkit. [sam: See gentoolkit side at https://github.com/gentoo/gentoolkit/pull/35]. Bug: https://bugs.gentoo.org/900224 Signed-off-by: Siddhanth Rathod gmail.com> Closes: https://github.com/gentoo/portage/pull/1123 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/bintree.py| 60 +++-- lib/portage/tests/dbapi/test_bintree.py | 6 ++-- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 580ce2f290..7a4166c120 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -809,6 +809,7 @@ class binarytree: getbinpkg_refresh=False, add_repos=(), force_reindex=False, +invalid_errors=True, ): """ Populates the binarytree with package metadata. @@ -839,7 +840,8 @@ class binarytree: try: update_pkgindex = self._populate_local( reindex="pkgdir-index-trusted" not in self.settings.features -or force_reindex +or force_reindex, +invalid_errors=invalid_errors, ) if update_pkgindex and self.dbapi.writable: @@ -882,7 +884,7 @@ class binarytree: self.populated = True -def _populate_local(self, reindex=True): +def _populate_local(self, reindex=True, invalid_errors=True): """ Populates the binarytree with local package metadata. @@ -1019,11 +1021,15 @@ class binarytree: self.dbapi.cpv_inject(mycpv) continue if not os.access(full_path, os.R_OK): -writemsg( -_("!!! Permission denied to read " "binary package: '%s'\n") -% full_path, -noiselevel=-1, -) +if invalid_errors: +writemsg( +_( +"!!! Permission denied to read " +"binary package: '%s'\n" +) +% full_path, +noiselevel=-1, +) self.invalids.append(myfile[:-5]) self.invalid_paths[myfile] = [full_path] continue @@ -1062,10 +1068,11 @@ class binarytree: binpkg_format=binpkg_format, ) except (PortagePackageException, SignatureException) as e: -writemsg( -f"!!! Invalid binary package: '{full_path}', {e}\n", -noiselevel=-1, -) +if invalid_errors: +writemsg( +f"!!! Invalid binary package: '{full_path}', {e}\n", +noiselevel=-1, +) self.invalid_paths[mypkg] = [full_path] continue mycat = pkg_metadata.get("CATEGORY", "") @@ -1073,10 +1080,11 @@ class binarytree: slot = pkg_metadata.get("SLOT", "") if not mycat or not mypf or not slot: # old-style or corrupt package -writemsg( -_("\n!!! Invalid binary package: '%s'\n") % full_path, -noiselevel=-1, -) +if invalid_errors: +writemsg( +_("\n!!! Invalid binary package: '%s'\n") % full_path, +noiselevel=-1, +) missing_keys = [] if not mycat: missing_keys.append("CATEGORY") @@ -1087,18 +1095,20 @@ class binarytree: msg = [] if missing_keys: missing_keys.sort() +if invalid_errors: +msg.append( +_("Missing metadata key(s): %s.") +% ", ".join(missing_keys) +) +if invalid_erro
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 2ccc08e3b629d3f56f028cd767ebd5ff1a8edaf1 Author: Zac Medico gentoo org> AuthorDate: Thu Oct 5 04:43:50 2023 + Commit: Zac Medico gentoo org> CommitDate: Thu Oct 5 05:18:58 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2ccc08e3 MergeProcess: Migrate to ForkProcess target parameter Bug: https://bugs.gentoo.org/915099 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/_MergeProcess.py | 73 +++--- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index 7c018222fe..012435dce3 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -1,6 +1,7 @@ -# Copyright 2010-2020 Gentoo Authors +# Copyright 2010-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +import functools import io import multiprocessing import platform @@ -73,6 +74,20 @@ class MergeProcess(ForkProcess): self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD") +self.target = functools.partial( +self._target, +lambda: self._counter, +lambda: self._elog_reader_fd, +lambda: self._dblink, +self.infloc, +self.mydbapi, +self.myebuild, +self.pkgloc, +self.prev_mtimes, +self.settings, +self.unmerge, +self.vartree.dbapi, +) super()._start() def _lock_vdb(self): @@ -197,13 +212,29 @@ class MergeProcess(ForkProcess): return pids -def _run(self): -os.close(self._elog_reader_fd) -counter = self._counter -mylink = self._dblink -portage.output.havecolor = not no_color(self.settings) +@staticmethod +def _target( +get_counter, +get_elog_reader_fd, +get_mylink, +infloc, +mydbapi, +myebuild, +pkgloc, +prev_mtimes, +settings, +unmerge, +vardb, +): +""" +TODO: Make all arguments picklable for the multiprocessing spawn start method. +""" +os.close(get_elog_reader_fd()) +counter = get_counter() +mylink = get_mylink() +portage.output.havecolor = not no_color(settings) # Avoid wastful updates of the vdb cache. -self.vartree.dbapi._flush_cache_enabled = False +vardb._flush_cache_enabled = False # In this subprocess we don't want PORTAGE_BACKGROUND to # suppress stdout/stderr output since they are pipes. We @@ -211,21 +242,21 @@ class MergeProcess(ForkProcess): # already be opened by the parent process, so we set the # "subprocess" value for use in conditional logging code # involving PORTAGE_LOG_FILE. -if not self.unmerge: +if not unmerge: # unmerge phases have separate logs -if self.settings.get("PORTAGE_BACKGROUND") == "1": -self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" +if settings.get("PORTAGE_BACKGROUND") == "1": +settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" else: -self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" -self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") -self.settings["PORTAGE_BACKGROUND"] = "subprocess" -self.settings.backup_changes("PORTAGE_BACKGROUND") +settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" +settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") +settings["PORTAGE_BACKGROUND"] = "subprocess" +settings.backup_changes("PORTAGE_BACKGROUND") rval = 1 -if self.unmerge: +if unmerge: if not mylink.exists(): rval = os.EX_OK -elif mylink.unmerge(ldpath_mtimes=self.prev_mtimes) == os.EX_OK: +elif mylink.unmerge(ldpath_mtimes=prev_mtimes) == os.EX_OK: mylink.lockdb() try: mylink.delete() @@ -234,11 +265,11 @@ class MergeProcess(ForkProcess): rval = os.EX_OK else: rval = mylink.merge( -self.pkgloc, -self.infloc, -myebuild=self.myebuild, -mydbapi=self.mydbapi, -prev_mtimes=self.prev_mtimes, +pkgloc, +infloc, +myebuild=myebuild, +mydbapi=mydbapi, +prev_mtimes=prev_mtimes, counter=counter, ) return rval
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: f45e2bb561f2f5d16bdf8f6cd31cc393d2794f92 Author: Zac Medico gentoo org> AuthorDate: Wed Oct 4 04:06:49 2023 + Commit: Zac Medico gentoo org> CommitDate: Wed Oct 4 04:11:14 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f45e2bb5 SyncfsProcess: Migrate to ForkProcess target parameter Bug: https://bugs.gentoo.org/915099 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/_SyncfsProcess.py | 15 +++ 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/_SyncfsProcess.py b/lib/portage/dbapi/_SyncfsProcess.py index 6beeac8dd4..ddc2240071 100644 --- a/lib/portage/dbapi/_SyncfsProcess.py +++ b/lib/portage/dbapi/_SyncfsProcess.py @@ -1,6 +1,8 @@ -# Copyright 2012 Gentoo Foundation +# Copyright 2012-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +import functools + from portage import os from portage.util._ctypes import find_library, LoadLibrary from portage.util._async.ForkProcess import ForkProcess @@ -16,6 +18,10 @@ class SyncfsProcess(ForkProcess): __slots__ = ("paths",) +def _start(self): +self.target = functools.partial(self._target, self._get_syncfs, self.paths) +super()._start() + @staticmethod def _get_syncfs(): filename = find_library("c") @@ -29,12 +35,13 @@ class SyncfsProcess(ForkProcess): return None -def _run(self): +@staticmethod +def _target(get_syncfs, paths): syncfs_failed = False -syncfs = self._get_syncfs() +syncfs = get_syncfs() if syncfs is not None: -for path in self.paths: +for path in paths: try: fd = os.open(path, os.O_RDONLY) except OSError:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: cec349d7cfcda634e3f76cd3ca0c1a89bc46f414 Author: hypersyd <70613804+siddhanthrathod users noreply github com> AuthorDate: Sat Sep 9 13:51:31 2023 + Commit: Sam James gentoo org> CommitDate: Tue Sep 26 21:09:28 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cec349d7 bintree: populate invalid_paths list for eclean-pkg Enable eclean-pkg to handle incomplete merges from binhost. It's an extension for gentoolkit's eclean-pkg clean invalids feature to deal with binpkgs created by the new binhost without any tar header or format due to incomplete merges. This results in gpkg.py --> class gpkg --> method _get_tar_format returning null which raises PortagePackageException and SignatureException in binpkg.py --> Class binarytree. Add an input for invalid paths api is being added so the eclean-pkg can clean it up. Invalids feature: https://github.com/gentoo/gentoolkit/commit/a16d0d4fbfb4614832c4b682b41284a9050af29f, https://github.com/gentoo/portage/commit/71daef3ac877329a0479a72ba333a9c801a36bf3 Bug: https://bugs.gentoo.org/900224 Signed-off-by: Siddhanth Rathod gmail.com> Closes: https://github.com/gentoo/portage/pull/1091 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/bintree.py | 9 + 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 42912b2eb5..580ce2f290 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1050,6 +1050,10 @@ class binarytree: else: binpkg_format = "gpkg" +for ext in SUPPORTED_XPAK_EXTENSIONS + SUPPORTED_GPKG_EXTENSIONS: +if myfile.endswith(ext): +mypkg = myfile[: -len(ext)] +break try: pkg_metadata = self._read_metadata( full_path, @@ -1062,14 +1066,11 @@ class binarytree: f"!!! Invalid binary package: '{full_path}', {e}\n", noiselevel=-1, ) +self.invalid_paths[mypkg] = [full_path] continue mycat = pkg_metadata.get("CATEGORY", "") mypf = pkg_metadata.get("PF", "") slot = pkg_metadata.get("SLOT", "") -for ext in SUPPORTED_XPAK_EXTENSIONS + SUPPORTED_GPKG_EXTENSIONS: -if myfile.endswith(ext): -mypkg = myfile[: -len(ext)] -break if not mycat or not mypf or not slot: # old-style or corrupt package writemsg(
[gentoo-commits] proj/portage:master commit in: /, lib/portage/dbapi/
commit: 5996c7bec3fe69e3ca69805777acdd8c5437b2ae Author: gcarq protonmail com> AuthorDate: Mon Sep 4 17:01:56 2023 + Commit: Sam James gentoo org> CommitDate: Tue Sep 26 20:54:12 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5996c7be vartree: Remove unused variables and parameters Removes unused local variable cache_incomplete in aux_get() and removes unused parameters for methods around counter_tick(). Signed-off-by: Michael Egger protonmail.com> Closes: https://github.com/gentoo/portage/pull/1089 Signed-off-by: Sam James gentoo.org> NEWS | 3 +++ lib/portage/dbapi/vartree.py | 24 ++-- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/NEWS b/NEWS index 340516a9b4..31a188e8f6 100644 --- a/NEWS +++ b/NEWS @@ -26,6 +26,9 @@ Bug fixes: * fetch: fix fetching of layout.conf when FEATURES=force-mirror (bug #877793). +Cleanups: +* vartree: Remove unused variables and parameters + portage-3.0.51 (2023-08-20) -- diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index d4b510082c..3f39e2b787 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -395,7 +395,7 @@ class vardbapi(dbapi): def cpv_inject(self, mycpv): "injects a real package into our on-disk database; assumes mycpv is valid and doesn't already exist" ensure_dirs(self.getpath(mycpv)) -counter = self.counter_tick(mycpv=mycpv) +counter = self.counter_tick() # write local package counter so that emerge clean does the right thing write_atomic(self.getpath(mycpv, filename="COUNTER"), str(counter)) @@ -794,7 +794,6 @@ class vardbapi(dbapi): pull_me = cache_these.union(wants) mydata = {"_mtime_": mydir_mtime} cache_valid = False -cache_incomplete = False cache_mtime = None metadata = None if pkg_data is not None: @@ -1141,13 +1140,10 @@ class vardbapi(dbapi): log_path=settings.get("PORTAGE_LOG_FILE"), ) -def counter_tick(self, myroot=None, mycpv=None): -""" -@param myroot: ignored, self._eroot is used instead -""" -return self.counter_tick_core(incrementing=1, mycpv=mycpv) +def counter_tick(self) -> int: +return self.counter_tick_core(incrementing=1) -def get_counter_tick_core(self, myroot=None, mycpv=None): +def get_counter_tick_core(self) -> int: """ Use this method to retrieve the counter instead of having to trust the value of a global counter @@ -1165,10 +1161,7 @@ class vardbapi(dbapi): it also corresponds to the total number of installation actions that have occurred in the history of this package database. - -@param myroot: ignored, self._eroot is used instead """ -del myroot counter = -1 try: with open( @@ -1219,7 +1212,7 @@ class vardbapi(dbapi): return max_counter + 1 -def counter_tick_core(self, myroot=None, incrementing=1, mycpv=None): +def counter_tick_core(self, incrementing: int = 1) -> int: """ This method will grab the next COUNTER value and record it back to the global file. Note that every package install must have @@ -1227,13 +1220,8 @@ class vardbapi(dbapi): into the same SLOT and in that case it's important that both packages have different COUNTER metadata. -@param myroot: ignored, self._eroot is used instead -@param mycpv: ignored -@rtype: int @return: new counter value """ -myroot = None -mycpv = None self.lock() try: counter = self.get_counter_tick_core() - 1 @@ -4974,7 +4962,7 @@ class dblink: # write local package counter for recording if counter is None: -counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv) +counter = self.vartree.dbapi.counter_tick() with open( _unicode_encode( os.path.join(self.dbtmpdir, "COUNTER"),
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 6293f7ba670fbfae5fd4bb6f4ba9fe3f822d19c3 Author: Sam James gentoo org> AuthorDate: Sat Sep 23 22:49:29 2023 + Commit: Sam James gentoo org> CommitDate: Sat Sep 23 22:49:49 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6293f7ba vartree: fix syntax Fixes: 1fc674667de01944269dfeccd70472facf83e7ba Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 878914ef06..d4b510082c 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -2687,7 +2687,7 @@ class dblink: unmerge_orphans = "unmerge-orphans" in self.settings.features calc_prelink = "prelink-checksums" in self.settings.features -pkgfiles = self.getcontents() if not pkgfiles +pkgfiles = pkgfiles if pkgfiles else self.getcontents() if pkgfiles: self.updateprotect() mykeys = list(pkgfiles)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 1fc674667de01944269dfeccd70472facf83e7ba Author: Sam James gentoo org> AuthorDate: Sat Sep 23 22:38:23 2023 + Commit: Sam James gentoo org> CommitDate: Sat Sep 23 22:38:31 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1fc67466 vartree: don't clobber pkgfiles Fixes: 743722ccfa234dd9d4a54f7fbfb14cc2ddf6f0f4 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 35a401b33c..878914ef06 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -2687,7 +2687,7 @@ class dblink: unmerge_orphans = "unmerge-orphans" in self.settings.features calc_prelink = "prelink-checksums" in self.settings.features -pkgfiles = self.getcontents() +pkgfiles = self.getcontents() if not pkgfiles if pkgfiles: self.updateprotect() mykeys = list(pkgfiles)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 743722ccfa234dd9d4a54f7fbfb14cc2ddf6f0f4 Author: Sam James gentoo org> AuthorDate: Sat Sep 23 22:27:37 2023 + Commit: Sam James gentoo org> CommitDate: Sat Sep 23 22:31:31 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=743722cc vartree: drop 'Grabbing a set' useless noise Prompted by genr8eofl asking about culling it. Fascinatingly, both the message *and* people agreeing it's futile goes back many years: * bf01d6c02087c2b363617ce6ac208d09ea81de25 ('don't display useless messages when using --quiet') * d9fc4acc572c6647a4f27b838d35d27d805d190e (svn migration) * 4ba2baf8456c0eeda0df1bf52a1a3d712d5ac9b7 ('Portage-2.0.41 release. LOTS of bug fixes. Check the ChangeLog.', 2002) This isn't useful for the user at all, so drop it. Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 5 + 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 08ea9234f6..35a401b33c 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -2660,10 +2660,6 @@ class dblink: ignored_unlink_errnos = self._ignored_unlink_errnos ignored_rmdir_errnos = self._ignored_rmdir_errnos -if not pkgfiles: -showMessage(_("No package files given... Grabbing a set.\n")) -pkgfiles = self.getcontents() - if others_in_slot is None: others_in_slot = [] slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot @@ -2691,6 +2687,7 @@ class dblink: unmerge_orphans = "unmerge-orphans" in self.settings.features calc_prelink = "prelink-checksums" in self.settings.features +pkgfiles = self.getcontents() if pkgfiles: self.updateprotect() mykeys = list(pkgfiles)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 03489ca1880d4429c18cf7da2ed27ae65a21510b Author: Andreas K. Hüttel gentoo org> AuthorDate: Tue Sep 19 18:15:30 2023 + Commit: Mike Gilbert gentoo org> CommitDate: Wed Sep 20 18:02:20 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=03489ca1 Catch error when trust helper is not found Closes: https://github.com/gentoo/portage/pull/1097 Signed-off-by: Andreas K. Hüttel gentoo.org> Signed-off-by: Mike Gilbert gentoo.org> lib/portage/dbapi/bintree.py | 12 +++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 9d7c45577b..42912b2eb5 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1245,7 +1245,17 @@ class binarytree: portage_trust_helper = self.settings.get("PORTAGE_TRUST_HELPER", "") if portage_trust_helper == "": return -ret = subprocess.run(portage_trust_helper) +try: +ret = subprocess.run(portage_trust_helper) +except FileNotFoundError: +writemsg( +_( +"\n!!! Portage trust helper %s for binary packages not found\n!!! Continuing, but did you install app-portage/getuto?\n" +) +% portage_trust_helper, +noiselevel=-1, +) +return ret.check_returncode() def _populate_remote(self, getbinpkg_refresh=True):
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: b2c91af6fb2dfdd227caf9a2eff588f02d2bf9fc Author: Oskari Pirhonen gmail com> AuthorDate: Tue Sep 5 01:21:22 2023 + Commit: Sam James gentoo org> CommitDate: Fri Sep 15 04:28:23 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2c91af6 vartree: keep build dir if postinst fails Bug: https://bugs.gentoo.org/704866 Signed-off-by: Oskari Pirhonen gmail.com> Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 7 +-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 7d1bba712e..08ea9234f6 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -6070,8 +6070,11 @@ class dblink: ebuild_phase.wait() self._elog_process() -if "noclean" not in self.settings.features and ( -retval == os.EX_OK or "fail-clean" in self.settings.features +# Keep the build dir around if postinst fails (bug #704866) +if ( +not self._postinst_failure +and "noclean" not in self.settings.features +and (retval == os.EX_OK or "fail-clean" in self.settings.features) ): if myebuild is None: myebuild = os.path.join(inforoot, self.pkg + ".ebuild")
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 7c0086fc0ab60cf0a726e3c88b11a21e147f3aa2 Author: Andrew Udvare gmail com> AuthorDate: Sun Jul 16 21:42:58 2023 + Commit: Sam James gentoo org> CommitDate: Sat Jul 29 03:57:45 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c0086fc dbapi: add some typing Signed-off-by: Andrew Udvare gmail.com> Closes: https://github.com/gentoo/portage/pull/1069 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/__init__.py | 27 +++ lib/portage/dbapi/porttree.py | 34 -- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py index 31453d149..428e4a48e 100644 --- a/lib/portage/dbapi/__init__.py +++ b/lib/portage/dbapi/__init__.py @@ -5,6 +5,7 @@ __all__ = ["dbapi"] import re import warnings +from typing import Any, Dict, List, Optional, Sequence, Tuple import portage @@ -29,7 +30,7 @@ from _emerge.Package import Package class dbapi: _category_re = re.compile(r"^\w[-.+\w]*$", re.UNICODE) -_categories = None +_categories: Optional[Tuple[str, ...]] = None _use_mutable = False _known_keys = frozenset(auxdbkeys) _pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository") @@ -38,7 +39,7 @@ class dbapi: pass @property -def categories(self): +def categories(self) -> Tuple[str, ...]: """ Use self.cp_all() to generate a category list. Mutable instances can delete the self._categories attribute in cases when the cached @@ -52,11 +53,11 @@ class dbapi: def close_caches(self): pass -def cp_list(self, cp, use_cache=1): +def cp_list(self, cp: str, use_cache: int = 1) -> Any: raise NotImplementedError(self) @staticmethod -def _cmp_cpv(cpv1, cpv2): +def _cmp_cpv(cpv1, cpv2) -> int: result = vercmp(cpv1.version, cpv2.version) if result == 0 and cpv1.build_time is not None and cpv2.build_time is not None: result = (cpv1.build_time > cpv2.build_time) - ( @@ -65,7 +66,7 @@ class dbapi: return result @staticmethod -def _cpv_sort_ascending(cpv_list): +def _cpv_sort_ascending(cpv_list: Sequence[Any]) -> None: """ Use this to sort self.cp_list() results in ascending order. It sorts in place and returns None. @@ -76,7 +77,7 @@ class dbapi: # dict to map strings back to their original values. cpv_list.sort(key=cmp_sort_key(dbapi._cmp_cpv)) -def cpv_all(self): +def cpv_all(self) -> List[str]: """Return all CPVs in the db Args: None @@ -93,16 +94,18 @@ class dbapi: cpv_list.extend(self.cp_list(cp)) return cpv_list -def cp_all(self, sort=False): +def cp_all(self, sort: bool = False) -> List[str]: """Implement this in a child class Args sort - return sorted results Returns: A list of strings 1 per CP in the datastore """ -return NotImplementedError +raise NotImplementedError -def aux_get(self, mycpv, mylist, myrepo=None): +def aux_get( +self, mycpv: str, mylist: str, myrepo: Optional[str] = None +) -> List[str]: """Return the metadata keys in mylist for mycpv Args: mycpv - "sys-apps/foo-1.0" @@ -114,7 +117,7 @@ class dbapi: """ raise NotImplementedError -def aux_update(self, cpv, metadata_updates): +def aux_update(self, cpv: str, metadata_updates: Dict[str, Any]) -> None: """ Args: cpv - "sys-apps/foo-1.0" @@ -124,7 +127,7 @@ class dbapi: """ raise NotImplementedError -def match(self, origdep, use_cache=1): +def match(self, origdep: str, use_cache: int = 1): """Given a dependency, try to find packages that match Args: origdep - Depend atom @@ -138,7 +141,7 @@ class dbapi: self._iter_match(mydep, self.cp_list(mydep.cp, use_cache=use_cache)) ) -def _iter_match(self, atom, cpv_iter): +def _iter_match(self, atom: str, cpv_iter): cpv_iter = iter(match_from_list(atom, cpv_iter)) if atom.repo: cpv_iter = self._iter_match_repo(atom, cpv_iter) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 3c38e99d4..c47b66bda 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -49,6 +49,7 @@ import functools import collections from collections import OrderedDict +from typing import List, Optional, Sequence, Type, Tuple, Union from urllib.parse import urlparse @@ -435,7 +436,9 @@ class portdbapi(dbapi): return license_path return None -def findname(self, mycpv, mytree=None, myrepo=None): +def findname( +self, mycpv: str, mytree: O
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/, man/
commit: da70f89cd88968501ca07cfaca6d73665d6e767a Author: Ulrich Müller gentoo org> AuthorDate: Tue May 23 09:08:36 2023 + Commit: Sam James gentoo org> CommitDate: Fri May 26 01:33:06 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=da70f89c Drop ignore-mtime from features The ignore-mtime feature violates PMS: https://projects.gentoo.org/pms/8/pms.html#x1-14500013.3.2 Allowing the package manager to arbitrarily modify timestamps of installed files will lead to problems with several languages: - Lisp (*.fasl must be newer than corresponding *.lisp) - Emacs (*.elc must not be older than *.el) - ghdl/VHDL (libraries check their mtimes) - Python (*.pyc vs *.py, not sure if this one is still an issue) If the PM does not preserve timestamps, there is no reasonable way to work around this on the ebuild level. (Some horrible hacks existed in the past, see for example impl-*-timestamp-hack in previous versions of common-lisp-common*.eclass which used to overwrite its installed files in pkg_postinst.) This partially reverts commit 89703c688868c9eb8cd6115cb42ff92f0b9668b8. Closes: https://bugs.gentoo.org/906978 See-also: https://bugs.gentoo.org/264130 Signed-off-by: Ulrich Müller gentoo.org> Signed-off-by: Sam James gentoo.org> lib/portage/const.py | 1 - lib/portage/dbapi/vartree.py | 14 ++ man/make.conf.5 | 6 -- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/lib/portage/const.py b/lib/portage/const.py index cb6796164..10a208ceb 100644 --- a/lib/portage/const.py +++ b/lib/portage/const.py @@ -159,7 +159,6 @@ SUPPORTED_FEATURES = frozenset( "getbinpkg", "gpg-keepalive", "icecream", -"ignore-mtime", "installsources", "ipc-sandbox", "keeptemp", diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 441e74661..dac350880 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -5422,7 +5422,6 @@ class dblink: srcroot = normalize_path(srcroot).rstrip(sep) + sep destroot = normalize_path(destroot).rstrip(sep) + sep calc_prelink = "prelink-checksums" in self.settings.features -ignore_mtime = "ignore-mtime" in self.settings.features protect_if_modified = ( "config-protect-if-modified" in self.settings.features @@ -5831,13 +5830,12 @@ class dblink: hardlink_candidates.append(mydest) zing = ">>>" else: -if not ignore_mtime: -mymtime = thismtime if thismtime is not None else mymtime -try: -os.utime(mydest, ns=(mymtime, mymtime)) -except OSError: -# utime can fail here with EPERM -pass +mymtime = thismtime if thismtime is not None else mymtime +try: +os.utime(mydest, ns=(mymtime, mymtime)) +except OSError: +# utime can fail here with EPERM +pass zing = "===" try: diff --git a/man/make.conf.5 b/man/make.conf.5 index ed5b7d7fc..85ee88c05 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -533,12 +533,6 @@ If your GPG is auto unlocked on login, you do not need this. .B icecream Enable portage support for the icecream package. .TP -.B ignore\-mtime -Do not update mtime if the target file is equal. This can be useful on some -filesystems to better utilize features like snapshots and data deduplication, -however this violates the preservation of file modification times as stipulated -in the PMS. -.TP .B installsources Install source code into /usr/src/debug/${CATEGORY}/${PF} (also see \fBsplitdebug\fR). This feature works only if debugedit is installed, CFLAGS
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 6da3e0fd0ddf551cf39b6ac329ee51051a22 Author: gcarq protonmail com> AuthorDate: Mon Mar 27 13:47:52 2023 + Commit: Sam James gentoo org> CommitDate: Tue May 23 00:22:09 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6da3e0fd mergeme: Put xattr comparison logic behind xattr feature flag Signed-off-by: gcarq protonmail.com> Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 676d4aa05..739b47f7f 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -6267,9 +6267,10 @@ class dblink: if mydmode is None or not stat.S_ISREG(mydmode) or mymode != mydmode: return True -excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "") -if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs): -return True +if "xattr" in self.settings.features: +excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "") +if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs): +return True return not filecmp.cmp(mysrc, mydest, shallow=False)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: a87be47f7d3245050da43d7c3ab4760d47e9fac5 Author: gcarq protonmail com> AuthorDate: Tue Feb 21 00:04:26 2023 + Commit: Sam James gentoo org> CommitDate: Tue May 23 00:22:09 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a87be47f mergeme: Don't overwrite files if the content matches Uses filecmp.cmp(shallow=False) to compare file contents and doesn't replace them if they are equal. This results in less disk churn and helps to keep filesystem snapshots as small as possible. Closes: https://bugs.gentoo.org/722270 Signed-off-by: gcarq protonmail.com> Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 47 +--- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index a9e332a74..327b72bed 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -3,6 +3,8 @@ __all__ = ["vardbapi", "vartree", "dblink"] + ["write_contents", "tar_contents"] +import filecmp + import portage portage.proxy.lazyimport.lazyimport( @@ -5800,28 +5802,33 @@ class dblink: # whether config protection or not, we merge the new file the # same way. Unless moveme=0 (blocking directory) if moveme: -# Create hardlinks only for source files that already exist -# as hardlinks (having identical st_dev and st_ino). -hardlink_key = (mystat.st_dev, mystat.st_ino) +# only replace the existing file if it differs, see #722270 +already_merged = os.path.exists(mydest) +if already_merged and filecmp.cmp(mysrc, mydest, shallow=False): +zing = "===" +else: +# Create hardlinks only for source files that already exist +# as hardlinks (having identical st_dev and st_ino). +hardlink_key = (mystat.st_dev, mystat.st_ino) -hardlink_candidates = self._hardlink_merge_map.get(hardlink_key) -if hardlink_candidates is None: -hardlink_candidates = [] -self._hardlink_merge_map[hardlink_key] = hardlink_candidates +hardlink_candidates = self._hardlink_merge_map.get(hardlink_key) +if hardlink_candidates is None: +hardlink_candidates = [] +self._hardlink_merge_map[hardlink_key] = hardlink_candidates -mymtime = movefile( -mysrc, -mydest, -newmtime=thismtime, -sstat=mystat, -mysettings=self.settings, -hardlink_candidates=hardlink_candidates, -encoding=_encodings["merge"], -) -if mymtime is None: -return 1 -hardlink_candidates.append(mydest) -zing = ">>>" +mymtime = movefile( +mysrc, +mydest, +newmtime=thismtime, +sstat=mystat, +mysettings=self.settings, +hardlink_candidates=hardlink_candidates, +encoding=_encodings["merge"], +) +if mymtime is None: +return 1 +hardlink_candidates.append(mydest) +zing = ">>>" try: self._merged_path(mydest, os.lstat(mydest))
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 35b5e4d71ebb5c7408dba7dc27cff0c22cad1562 Author: gcarq protonmail com> AuthorDate: Mon Mar 27 13:31:44 2023 + Commit: Sam James gentoo org> CommitDate: Tue May 23 00:22:09 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=35b5e4d7 mergeme: Rely on mydmode instead of calling os.path.exists again Signed-off-by: gcarq protonmail.com> Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 9 + 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 317cf327a..676d4aa05 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -5544,6 +5544,8 @@ class dblink: destmd5, mydest_link, ) +if protected and moveme: +mydmode = None zing = "!!!" if not moveme: @@ -5584,6 +5586,7 @@ class dblink: msg.append("") self._eerror("preinst", msg) mydest = newdest +mydmode = None # if secondhand is None it means we're operating in "force" mode and should not create a second hand. if (secondhand is not None) and (not os.path.exists(myrealto)): @@ -5797,6 +5800,7 @@ class dblink: msg.append("") self._eerror("preinst", msg) mydest = newdest +mydmode = None # whether config protection or not, we merge the new file the # same way. Unless moveme=0 (blocking directory) @@ -6260,10 +6264,7 @@ class dblink: Takes file mode and extended attributes into account. Should only be used for regular files. """ -if not os.path.exists(mydest): -return True - -if mymode != mydmode: +if mydmode is None or not stat.S_ISREG(mydmode) or mymode != mydmode: return True excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
[gentoo-commits] proj/portage:master commit in: /, lib/portage/dbapi/
commit: 2569a1a1d889a76af80ce24a005d54269df7d2e2 Author: Sheng Yu protonmail com> AuthorDate: Fri Apr 7 04:50:49 2023 + Commit: Sam James gentoo org> CommitDate: Fri Apr 7 09:49:11 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2569a1a1 dbapi: Catch invalid binpkg error during pkgmove Bug: https://bugs.gentoo.org/877271 Bug: https://bugs.gentoo.org/903917 Bug: https://bugs.gentoo.org/903926 Signed-off-by: Sheng Yu protonmail.com> Closes: https://github.com/gentoo/portage/pull/1022 Signed-off-by: Sam James gentoo.org> NEWS | 3 +++ lib/portage/dbapi/__init__.py | 8 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 506e673e1..1d05f1de2 100644 --- a/NEWS +++ b/NEWS @@ -60,6 +60,9 @@ Features: for a time for compatibility. Bug fixes: +* dbapi: Handle mismatched binpkg structure during pkgmoves (bug #877271, + bug #903917, bug #903926). + * tests: util/test_shelve: fix test failure if the backend for the shelve module does not create the shelve db using the literal filename. diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py index 366a6c170..31453d149 100644 --- a/lib/portage/dbapi/__init__.py +++ b/lib/portage/dbapi/__init__.py @@ -4,6 +4,7 @@ __all__ = ["dbapi"] import re +import warnings import portage @@ -21,7 +22,7 @@ from portage.const import MERGING_IDENTIFIER from portage import os from portage import auxdbkeys from portage.eapi import _get_eapi_attrs -from portage.exception import InvalidData +from portage.exception import InvalidBinaryPackageFormat, InvalidData from portage.localization import _ from _emerge.Package import Package @@ -410,7 +411,10 @@ class dbapi: updates_list, metadata, parent=pkg ) if metadata_updates: -aux_update(cpv, metadata_updates) +try: +aux_update(cpv, metadata_updates) +except InvalidBinaryPackageFormat as e: +warnings.warn(e) if onUpdate: onUpdate(maxval, i + 1) if onProgress:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/resolver/, /, lib/portage/_emirrordist/, ...
commit: 0431aa7db28e69af309a8175ec9e13a23439e2fa Author: Siddhanth Rathod gmail com> AuthorDate: Sat Mar 4 19:29:28 2023 + Commit: Sam James gentoo org> CommitDate: Tue Mar 21 02:35:55 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0431aa7d emerge: respect NO_COLOR Bug: https://bugs.gentoo.org/898224 Signed-off-by: Siddhanth Rathod gmail.com> Closes: https://github.com/gentoo/portage/pull/1003 Signed-off-by: Sam James gentoo.org> NEWS | 7 ++- bin/ebuild | 7 +++ bin/egencache | 9 +++-- bin/etc-update | 1 + bin/isolated-functions.sh | 11 +-- bin/portageq | 6 ++ bin/save-ebuild-env.sh | 2 +- lib/_emerge/AbstractEbuildProcess.py | 7 +++ lib/_emerge/actions.py | 12 ++-- lib/portage/_emirrordist/main.py | 2 +- lib/portage/dbapi/_MergeProcess.py | 5 ++--- lib/portage/package/ebuild/_config/special_env_vars.py | 2 ++ lib/portage/package/ebuild/_ipc/QueryCommand.py| 6 ++ lib/portage/tests/__init__.py | 3 ++- lib/portage/tests/resolver/ResolverPlayground.py | 4 ++-- lib/portage/tests/runTests.py | 2 +- lib/portage/util/__init__.py | 11 +++ 17 files changed, 57 insertions(+), 40 deletions(-) diff --git a/NEWS b/NEWS index 03e262c69..01209c9fb 100644 --- a/NEWS +++ b/NEWS @@ -48,7 +48,12 @@ Cleanups: in terms of priority for providers. Features: -* TODO +* Respect the NO_COLOR environment variable for disabling color. + + The new quasi-standard for disabling ANSI color is to check for the NO_COLOR + environment variable and disable color when the variable has a nonempty value. + See bug #898224. Portage previously used NOCOLOR. It continues to support NOCOLOR + for a time for compatibility. Bug fixes: * TODO diff --git a/bin/ebuild b/bin/ebuild index 8d1908795..8fccbea89 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -67,7 +67,6 @@ try: from _emerge.RootConfig import RootConfig portage.process.sanitize_fds() - description = "See the ebuild(1) man page for more info" usage = "Usage: ebuild [command] ..." parser = argparse.ArgumentParser(description=description, usage=usage) @@ -128,14 +127,14 @@ try: if not opts.color == "y" and ( opts.color == "n" -or portage.settings.get("NOCOLOR") in ("yes", "true") +or portage.util.no_color(portage.settings) or portage.settings.get("TERM") == "dumb" or not sys.stdout.isatty() ): portage.output.nocolor() portage.settings.unlock() -portage.settings["NOCOLOR"] = "true" -portage.settings.backup_changes("NOCOLOR") +portage.settings["NO_COLOR"] = "true" +portage.settings.backup_changes("NO_COLOR") portage.settings.lock() apply_priorities(portage.settings) diff --git a/bin/egencache b/bin/egencache index 1a6118bde..671df3014 100755 --- a/bin/egencache +++ b/bin/egencache @@ -67,7 +67,7 @@ try: from portage.package.ebuild._parallel_manifest.ManifestScheduler import ( ManifestScheduler, ) -from portage.util import cmp_sort_key, writemsg_level +from portage.util import cmp_sort_key, writemsg_level, no_color from portage.util._async.AsyncFunction import AsyncFunction from portage.util._async.run_main_scheduler import run_main_scheduler from portage.util._async.TaskScheduler import TaskScheduler @@ -1105,12 +1105,9 @@ try: if "PATH" in os.environ: env["PATH"] = os.environ["PATH"] -if not sys.stdout.isatty() or os.environ.get("NOCOLOR", "").lower() in ( -"yes", -"true", -): +if not sys.stdout.isatty() or no_color(os.environ): portage.output.nocolor() -env["NOCOLOR"] = "true" +env["NO_COLOR"] = "true" parser, options, atoms = parse_args(args) diff --git a/bin/etc-update b/bin/etc-update index 59e709168..14bd80b84 100755 --- a/bin/etc-update +++ b/bin/etc-update @@ -798,6 +798,7 @@ portage_vars=( EROOT USERLAND NOCOLOR + NO_COLOR ) if type -P portageq > /dev/null; then diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 882789132..06be030fb 100644 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -434,14 +434,21 @@ RC_INDENTATION='' RC_DEFAULT_INDENT=2 RC_DOT_PATTERN='' -case "${NOCOLOR:-false}" in + + +if [[ -z ${NO_COLOR} ]] ; then + case ${NOCOLOR:-false} in yes|true) __unset_co
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/update/
commit: c00dbfb9d8e3c494173200436ec4c5bc2689707f Author: Sheng Yu protonmail com> AuthorDate: Fri Feb 10 07:55:26 2023 + Commit: Sam James gentoo org> CommitDate: Fri Feb 17 01:23:14 2023 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c00dbfb9 Ignore update GPG signed binpkg User cannot update them without key. Signed-off-by: Sheng Yu protonmail.com> Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/bintree.py | 8 lib/portage/tests/update/test_move_ent.py | 5 - 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 23bc41d44..5f58c548d 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -674,6 +674,14 @@ class binarytree: elif binpkg_format == "gpkg": mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, binpkg_path) mydata = mybinpkg.get_metadata() +if mybinpkg.signature_exist: +writemsg( +colorize( +"WARN", +f"Binpkg update ignored for signed package: {binpkg_path}", +) +) +continue decode_metadata_name = True else: continue diff --git a/lib/portage/tests/update/test_move_ent.py b/lib/portage/tests/update/test_move_ent.py index 562d7b107..22d0c8feb 100644 --- a/lib/portage/tests/update/test_move_ent.py +++ b/lib/portage/tests/update/test_move_ent.py @@ -58,7 +58,10 @@ class MoveEntTestCase(TestCase): ebuilds=ebuilds, installed=installed, user_config={ -"make.conf": (f'BINPKG_FORMAT="{binpkg_format}"',), +"make.conf": ( +f'BINPKG_FORMAT="{binpkg_format}"', +'FEATURES="-binpkg-signing"', +), }, )
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 2f020ae6aca04a0db68b7ceb5413e2b815d2169d Author: Sam James gentoo org> AuthorDate: Wed Dec 21 01:30:22 2022 + Commit: Sam James gentoo org> CommitDate: Wed Dec 21 01:30:55 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2f020ae6 Revert "porttree: skip metadata verification if repository is immutable (not volatile)" This reverts commit f05140beb2dd9a3577ff2042941cb3e4fbb1df31. I didn't mean to push this one yet, which we're going to handle separately. Bug: https://bugs.gentoo.org/528394 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/porttree.py | 7 ++- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 448a7f300..15e1fd6ff 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -614,8 +614,7 @@ class portdbapi(dbapi): if ro_auxdb is not None: auxdbs.append(ro_auxdb) auxdbs.append(self.auxdb[repo_path]) -repo = self.repositories.get_repo_for_location(repo_path) -eclass_db = repo.eclass_db +eclass_db = self.repositories.get_repo_for_location(repo_path).eclass_db for auxdb in auxdbs: try: @@ -638,9 +637,7 @@ class portdbapi(dbapi): # EAPI from _parse_eapi_ebuild_head, we disregard cache entries # for unsupported EAPIs. continue -if not repo.volatile or auxdb.validate_entry( -metadata, ebuild_hash, eclass_db -): +if auxdb.validate_entry(metadata, ebuild_hash, eclass_db): break else: metadata = None
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: f05140beb2dd9a3577ff2042941cb3e4fbb1df31 Author: Sam James gentoo org> AuthorDate: Sat Dec 17 05:15:27 2022 + Commit: Sam James gentoo org> CommitDate: Wed Dec 21 01:28:02 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f05140be porttree: skip metadata verification if repository is immutable (not volatile) With Python 3.11, I get: - ~232695 total syscalls before (38268 read calls) - ~203656 total syscalls after (27381 read calls) It's a cheap improvement for simply promising to not touch ebuilds in a repository. Bug: https://bugs.gentoo.org/528394 See: a72a01746638debe472496bd8fc661992a6ba08b See: 5c7bf4eb09f644813a6f017ffd91665664142560 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/porttree.py | 7 +-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 15e1fd6ff..448a7f300 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -614,7 +614,8 @@ class portdbapi(dbapi): if ro_auxdb is not None: auxdbs.append(ro_auxdb) auxdbs.append(self.auxdb[repo_path]) -eclass_db = self.repositories.get_repo_for_location(repo_path).eclass_db +repo = self.repositories.get_repo_for_location(repo_path) +eclass_db = repo.eclass_db for auxdb in auxdbs: try: @@ -637,7 +638,9 @@ class portdbapi(dbapi): # EAPI from _parse_eapi_ebuild_head, we disregard cache entries # for unsupported EAPIs. continue -if auxdb.validate_entry(metadata, ebuild_hash, eclass_db): +if not repo.volatile or auxdb.validate_entry( +metadata, ebuild_hash, eclass_db +): break else: metadata = None
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, /
commit: 66f64d08ffb13ca24cfc11584fe667b1c391fbe8 Author: Sam James gentoo org> AuthorDate: Thu Dec 1 06:31:53 2022 + Commit: Sam James gentoo org> CommitDate: Thu Dec 1 07:08:02 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=66f64d08 vartree: use ewarn if merging despite collisions Use ewarn, not eerror, if merge is going ahead despite collisions. Collisions aren't always fatal - say if FEATURES="protect-owned unmerge-orphans" and the collision is over an orphaned file. This changes the colors used once we discover that all relevant files are orphaned to be warning rather than an error, to make it less scary and distinguish from other situations. Closes: https://github.com/gentoo/portage/pull/952 Signed-off-by: Sam James gentoo.org> NEWS | 4 +++- lib/portage/dbapi/vartree.py | 18 +++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index d64fcac91..a907526aa 100644 --- a/NEWS +++ b/NEWS @@ -2,7 +2,9 @@ portage-3.0.41 (UNRELEASED) -- Features: -* TODO +* vartree: On collisions which are non-fatal (e.g. orphaned files when using + FEATURES="protect-owned"), Portage now has friendlier output + by using ewarn rather than eerror. Bug fixes: * TODO diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index a95d60691..139424c0a 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -4187,6 +4187,9 @@ class dblink: def _eqawarn(self, phase, lines): self._elog("eqawarn", phase, lines) +def _ewarn(self, phase, lines): +self._elog("ewarn", phase, lines) + def _eerror(self, phase, lines): self._elog("eerror", phase, lines) @@ -4388,6 +4391,9 @@ class dblink: ], ) +def ewarn(lines): +self._ewarn("preinst", lines) + def eerror(lines): self._eerror("preinst", lines) @@ -4884,6 +4890,10 @@ class dblink: finally: self.unlockdb() +collision_message_type = ewarn +if collision_protect or protect_owned and owners: +collision_message_type = eerror + for pkg, owned_files in owners.items(): msg = [] msg.append(pkg_info_strs[pkg.mycpv]) @@ -4892,10 +4902,10 @@ class dblink: "\t%s" % os.path.join(destroot, f.lstrip(os.path.sep)) ) msg.append("") -eerror(msg) +collision_message_type(msg) if not owners: -eerror( +collision_message_type( [_("None of the installed" " packages claim the file(s)."), ""] ) @@ -4935,10 +4945,12 @@ class dblink: " If necessary, refer to your elog " "messages for the whole content of the above message." ) -eerror(wrap(msg, 70)) if abort: +eerror(wrap(msg, 70)) return 1 +else: +ewarn(wrap(msg, 70)) # The merge process may move files out of the image directory, # which causes invalidation of the .installed flag.
[gentoo-commits] proj/portage:master commit in: /, lib/portage/dbapi/
commit: d9699fa0b09a1d5a397edf0bb25c7662ac88e09d Author: Sheng Yu protonmail com> AuthorDate: Wed Nov 30 18:13:29 2022 + Commit: Sam James gentoo org> CommitDate: Wed Nov 30 22:28:58 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d9699fa0 Properly assign filenames for remote packages Signed-off-by: Sheng Yu protonmail.com> Closes: https://github.com/gentoo/portage/pull/950 Signed-off-by: Sam James gentoo.org> NEWS | 2 ++ lib/portage/dbapi/bintree.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 923b95a43..1cb2b2468 100644 --- a/NEWS +++ b/NEWS @@ -49,6 +49,8 @@ Bug fixes: * env-update: Also generate PATH definition in systemd user environment file /etc/environment.d/10-gentoo-env.conf +* bintree: Properly assign filenames for remote binpkgs. + * --disable-static is only passed for libtool-enabled configure scripts in EAPI 8. This avoids annoying warnings when a configure script has a flag such as --disable-static_link that would then trigger a QA warning (bug #814380). diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index d7c18e2e9..baf001c8f 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -2031,7 +2031,7 @@ class binarytree: filename = None build_id = None if allocate_new: -filename, build_id = self._allocate_filename(cpv) +filename, build_id = self._allocate_filename(cpv, remote_binpkg_format) elif self._is_specific_instance(cpv): instance_key = self.dbapi._instance_key(cpv) path = self._pkg_paths.get(instance_key)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: a7dd39c1ae4a5ea4e3252ae8129fbd671c95d5f7 Author: Sheng Yu protonmail com> AuthorDate: Tue Nov 8 22:52:46 2022 + Commit: Sam James gentoo org> CommitDate: Tue Nov 8 23:07:46 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7dd39c1 Do not drop default PATH in the package index (avoid Packages regeneration) The new Packages index (introduced in 445f10f4214c673f8fe0a9cc518c12767be4f159) needs PATH but it got dropped later on when processing, so we would try to rebuild the index on every emerge call. This fixes regenerating Packages loop. (Note that this didn't affect a released version.) Fixes: 445f10f4214c673f8fe0a9cc518c12767be4f159 Bug: https://bugs.gentoo.org/877357 Bug: https://bugs.gentoo.org/877419 Signed-off-by: Sheng Yu protonmail.com> Closes: https://github.com/gentoo/portage/pull/934 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/bintree.py | 21 +++-- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 771abedd5..d7c18e2e9 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1000,20 +1000,9 @@ class binarytree: pkg_paths[instance_key] = mypath # update the path if the package has been moved oldpath = d.get("PATH") -if oldpath and oldpath != mypath: -update_pkgindex = True -# Omit PATH if it is the default path for -# the current Packages format version. -if (mypath != mycpv + ".tbz2") and ( -mypath != mycpv + ".gpkg.tar" -): +if oldpath != mypath: d["PATH"] = mypath -if not oldpath: -update_pkgindex = True -else: -d.pop("PATH", None) -if oldpath: -update_pkgindex = True +update_pkgindex = True self.dbapi.cpv_inject(mycpv) continue if not os.access(full_path, os.R_OK): @@ -1218,11 +1207,7 @@ class binarytree: self.dbapi.cpv_remove(mycpv) del pkg_paths[_instance_key(mycpv)] -# record location if it's non-default -if (mypath != mycpv + ".tbz2") and (mypath != mycpv + ".gpkg.tar"): -d["PATH"] = mypath -else: -d.pop("PATH", None) +d["PATH"] = mypath metadata[_instance_key(mycpv)] = d if reindex:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/emerge/, lib/portage/tests/dbapi/
commit: 31c04f4b5b014a6a80ad4d1c2c8574106bab2c56 Author: David Palao gmail com> AuthorDate: Fri Sep 16 15:08:41 2022 + Commit: Mike Gilbert gentoo org> CommitDate: Sun Sep 25 19:10:34 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=31c04f4b improvement(bintree)!: binarytree.populate has new default getbinpkg_refresh=False This, together with a previous commit, fixes Bug #864259: sys-apps/portage: fetches binhost's 'Packages' file multiple times Bug: https://bugs.gentoo.org/864259 Signed-off-by: David Palao gmail.com> Signed-off-by: Mike Gilbert gentoo.org> lib/portage/dbapi/bintree.py | 2 +- lib/portage/tests/dbapi/test_bintree.py | 17 + lib/portage/tests/emerge/test_actions.py | 5 + 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index cea9378d5..c8bb2c88e 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -807,7 +807,7 @@ class binarytree: except PortageException: pass -def populate(self, getbinpkgs=False, getbinpkg_refresh=True, add_repos=()): +def populate(self, getbinpkgs=False, getbinpkg_refresh=False, add_repos=()): """ Populates the binarytree with package metadata. diff --git a/lib/portage/tests/dbapi/test_bintree.py b/lib/portage/tests/dbapi/test_bintree.py index 881d8ff48..4b4982a54 100644 --- a/lib/portage/tests/dbapi/test_bintree.py +++ b/lib/portage/tests/dbapi/test_bintree.py @@ -136,3 +136,20 @@ class BinarytreeTestCase(TestCase): ), noiselevel=-1, ) + +@patch("portage.dbapi.bintree.BinRepoConfigLoader") +@patch("portage.dbapi.bintree.binarytree._populate_remote") +@patch("portage.dbapi.bintree.binarytree._populate_local") +def test_default_getbinpkg_refresh_in_populate( +self, ppopulate_local, ppopulate_remote, pBinRepoConfigLoader +): +"""Bug #864259 +This test fixes the bug. It requires that +``_emerge.actions.run_action`` calls ``binarytree.populate`` +explicitly with ``getbinpkg_refresh=True`` +""" +settings = MagicMock() +settings.__getitem__.return_value = "/some/path" +bt = binarytree(pkgdir="/tmp", settings=settings) +bt.populate(getbinpkgs=True) +ppopulate_remote.assert_called_once_with(getbinpkg_refresh=False) diff --git a/lib/portage/tests/emerge/test_actions.py b/lib/portage/tests/emerge/test_actions.py index 014d39dd9..ee59cef5e 100644 --- a/lib/portage/tests/emerge/test_actions.py +++ b/lib/portage/tests/emerge/test_actions.py @@ -30,6 +30,11 @@ class RunActionTestCase(TestCase): papply, padjust, profile_ckeck): +"""Ensure that ``binarytree.populate`` API is correctly used. +The point of this test is to ensure that the ``populate`` method +is called as expected: since it is the first time that ``populate`` +is called, it must use ``getbinpkg_refresh=True``. +""" config = MagicMock() config.action = None config.opts = {"--quiet": True, "--usepkg": True}
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: a83507be7ce04d3ac421f9cbe8b63816809b0f4e Author: David Palao gmail com> AuthorDate: Fri Sep 23 13:42:33 2022 + Commit: Mike Gilbert gentoo org> CommitDate: Sun Sep 25 19:07:52 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a83507be chore(bintree): removed useless ``if True:`` This commit de-indents a large block. Signed-off-by: David Palao gmail.com> Signed-off-by: Mike Gilbert gentoo.org> lib/portage/dbapi/bintree.py | 263 +-- 1 file changed, 131 insertions(+), 132 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 7f5dc051c..28b3c481b 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -473,143 +473,142 @@ class binarytree: stacklevel=2, ) -if True: -self.pkgdir = normalize_path(pkgdir) -# NOTE: Event if binpkg-multi-instance is disabled, it's -# still possible to access a PKGDIR which uses the -# binpkg-multi-instance layout (or mixed layout). -self._multi_instance = "binpkg-multi-instance" in settings.features -if self._multi_instance: -self._allocate_filename = self._allocate_filename_multi -self.dbapi = bindbapi(self, settings=settings) -self.update_ents = self.dbapi.update_ents -self.move_slot_ent = self.dbapi.move_slot_ent -self.populated = 0 -self.tree = {} -self._binrepos_conf = None -self._remote_has_index = False -self._remotepkgs = None # remote metadata indexed by cpv -self._additional_pkgs = {} -self.invalids = [] -self.settings = settings -self._pkg_paths = {} -self._populating = False -self._all_directory = os.path.isdir(os.path.join(self.pkgdir, "All")) -self._pkgindex_version = 0 -self._pkgindex_hashes = ["MD5", "SHA1"] -self._pkgindex_file = os.path.join(self.pkgdir, "Packages") -self._pkgindex_keys = self.dbapi._aux_cache_keys.copy() -self._pkgindex_keys.update(["CPV", "SIZE"]) -self._pkgindex_aux_keys = [ -"BASE_URI", -"BDEPEND", +self.pkgdir = normalize_path(pkgdir) +# NOTE: Event if binpkg-multi-instance is disabled, it's +# still possible to access a PKGDIR which uses the +# binpkg-multi-instance layout (or mixed layout). +self._multi_instance = "binpkg-multi-instance" in settings.features +if self._multi_instance: +self._allocate_filename = self._allocate_filename_multi +self.dbapi = bindbapi(self, settings=settings) +self.update_ents = self.dbapi.update_ents +self.move_slot_ent = self.dbapi.move_slot_ent +self.populated = 0 +self.tree = {} +self._binrepos_conf = None +self._remote_has_index = False +self._remotepkgs = None # remote metadata indexed by cpv +self._additional_pkgs = {} +self.invalids = [] +self.settings = settings +self._pkg_paths = {} +self._populating = False +self._all_directory = os.path.isdir(os.path.join(self.pkgdir, "All")) +self._pkgindex_version = 0 +self._pkgindex_hashes = ["MD5", "SHA1"] +self._pkgindex_file = os.path.join(self.pkgdir, "Packages") +self._pkgindex_keys = self.dbapi._aux_cache_keys.copy() +self._pkgindex_keys.update(["CPV", "SIZE"]) +self._pkgindex_aux_keys = [ +"BASE_URI", +"BDEPEND", +"BINPKG_FORMAT", +"BUILD_ID", +"BUILD_TIME", +"CHOST", +"DEFINED_PHASES", +"DEPEND", +"DESCRIPTION", +"EAPI", +"FETCHCOMMAND", +"IDEPEND", +"IUSE", +"KEYWORDS", +"LICENSE", +"PDEPEND", +"PKGINDEX_URI", +"PROPERTIES", +"PROVIDES", +"RDEPEND", +"repository", +"REQUIRES", +"RESTRICT", +"RESUMECOMMAND", +"SIZE", +"SLOT", +"USE", +] +self._pkgindex_aux_keys = list(self._pkgindex_aux_keys) +self._pkgindex_use_evaluated_keys = ( +"BDEPEND", +"DEPEND", +"IDEPEND", +"LICENSE", +"RDEPEND", +"PDEPEND", +"PROPERTIES", +"RESTRICT", +) +self._pkgindex_header = None +self._pkgindex_header_keys = set( +[ +"ACCEPT_KEYWORDS", +"ACCEPT_LICENSE", +"ACCEPT_PROPERTIES", +"ACCEPT_RESTRICT", "BINPKG_FORMAT", -"BUILD_ID", -
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/package/ebuild/, ...
commit: fa901a6510c4a5f72dec6aad86db6fe7efd6e7b3 Author: Sheng Yu protonmail com> AuthorDate: Tue Sep 6 18:38:27 2022 + Commit: Michał Górny gentoo org> CommitDate: Fri Sep 9 10:16:06 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa901a65 Add BUILD_ID to metadata during binary packaging Also create placeholder for new multi-instance package before actually packaging to avoid race. Signed-off-by: Sheng Yu protonmail.com> Closes: https://github.com/gentoo/portage/pull/893 Signed-off-by: Michał Górny gentoo.org> bin/gpkg-helper.py | 4 +- bin/misc-functions.sh | 6 +- bin/quickpkg | 2 +- lib/_emerge/Binpkg.py | 2 +- lib/_emerge/BinpkgFetcher.py | 27 ++-- lib/_emerge/BinpkgPrefetcher.py| 8 +- lib/_emerge/EbuildBinpkg.py| 35 +++-- lib/_emerge/Scheduler.py | 2 +- lib/portage/dbapi/bintree.py | 145 +++-- .../package/ebuild/_config/special_env_vars.py | 1 + lib/portage/package/ebuild/doebuild.py | 3 +- 11 files changed, 126 insertions(+), 109 deletions(-) diff --git a/bin/gpkg-helper.py b/bin/gpkg-helper.py index d45177f3e..4752c84ee 100755 --- a/bin/gpkg-helper.py +++ b/bin/gpkg-helper.py @@ -19,7 +19,7 @@ def command_compose(args): sys.stderr.write("4 arguments are required, got %s\n" % len(args)) return 1 -cpv, binpkg_path, metadata_dir, image_dir = args +basename, binpkg_path, metadata_dir, image_dir = args if not os.path.isdir(metadata_dir): sys.stderr.write(usage) @@ -31,7 +31,7 @@ def command_compose(args): sys.stderr.write("Argument 4 is not a directory: '%s'\n" % image_dir) return 1 -gpkg_file = portage.gpkg.gpkg(portage.settings, cpv, binpkg_path) +gpkg_file = portage.gpkg.gpkg(portage.settings, basename, binpkg_path) metadata = gpkg_file._generate_metadata_from_dir(metadata_dir) gpkg_file.compress(image_dir, metadata) return os.EX_OK diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index faa8184c6..170e60d1c 100755 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -512,6 +512,10 @@ __dyn_package() { die "PORTAGE_BINPKG_TMPFILE is unset" mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed" + if [[ ! -z "${BUILD_ID}" ]]; then + echo -n "${BUILD_ID}" > "${PORTAGE_BUILDDIR}"/build-info/BUILD_ID + fi + if [[ "${BINPKG_FORMAT}" == "xpak" ]]; then local tar_options="" @@ -547,7 +551,7 @@ __dyn_package() { elif [[ "${BINPKG_FORMAT}" == "gpkg" ]]; then PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}"/gpkg-helper.py compress \ - "${CATEGORY}/${PF}" "${PORTAGE_BINPKG_TMPFILE}" "${PORTAGE_BUILDDIR}/build-info" "${D}" + "${PF}${BUILD_ID:+-${BUILD_ID}}" "${PORTAGE_BINPKG_TMPFILE}" "${PORTAGE_BUILDDIR}/build-info" "${D}" if [[ $? -ne 0 ]]; then rm -f "${PORTAGE_BINPKG_TMPFILE}" die "Failed to create binpkg file" diff --git a/bin/quickpkg b/bin/quickpkg index 773c1c07e..eef5f912f 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -206,7 +206,7 @@ def quickpkg_atom(options, infos, arg, eout): finally: if have_lock: dblnk.unlockdb() -pkg_info = bintree.inject(cpv, filename=binpkg_tmpfile) +pkg_info = bintree.inject(cpv, current_pkg_path=binpkg_tmpfile) # The pkg_info value ensures that the following getname call # returns the correct path when FEATURES=binpkg-multi-instance # is enabled, but fallback to cpv in case the inject call diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py index 15eb56092..949ac8ee7 100644 --- a/lib/_emerge/Binpkg.py +++ b/lib/_emerge/Binpkg.py @@ -246,7 +246,7 @@ class Binpkg(CompositeTask): if self._fetched_pkg: pkg_path = self._bintree.getname( -self._bintree.inject(pkg.cpv, filename=self._fetched_pkg), +self._bintree.inject(pkg.cpv, current_pkg_path=self._fetched_pkg), allocate_new=False, ) else: diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index d5275ea11..b7021e276 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -12,6 +12,7 @@ import sys import portage from portage import os from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS +from portage.const import SUPPORTED_XPAK_EXTENSIONS, SUPPORTED_GPKG_EXTENSIONS from portage.exception import FileNotFound, InvalidBinaryPackageForma
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 52411290c67535d94c7b20fa996ae7108014adfb Author: Sheng Yu protonmail com> AuthorDate: Fri Aug 19 20:24:59 2022 + Commit: Michał Górny gentoo org> CommitDate: Fri Sep 9 10:15:57 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=52411290 Detect binary package format if not in database Signed-off-by: Sheng Yu protonmail.com> Signed-off-by: Michał Górny gentoo.org> lib/portage/dbapi/bintree.py | 6 ++ 1 file changed, 6 insertions(+) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index b441fff9a..0857ff21a 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1039,6 +1039,12 @@ class binarytree: if match: binpkg_format = match.get("BINPKG_FORMAT", None) +if not binpkg_format: +if myfile.endswith(SUPPORTED_XPAK_EXTENSIONS): +binpkg_format = "xpak" +elif myfile.endswith(SUPPORTED_GPKG_EXTENSIONS): +binpkg_format = "gpkg" + if gpkg_only: if binpkg_format != "gpkg": if not gpkg_only_warned:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/gpkg/, lib/portage/
commit: 1d94992a2df2b5cc963c26c7978a899dc642deb1 Author: Sheng Yu protonmail com> AuthorDate: Thu Sep 1 14:44:55 2022 + Commit: Michał Górny gentoo org> CommitDate: Fri Sep 9 10:15:58 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1d94992a Move all files into basename/DATA structure Signed-off-by: Sheng Yu protonmail.com> Signed-off-by: Michał Górny gentoo.org> lib/portage/binpkg.py| 8 +- lib/portage/dbapi/bintree.py | 2 +- lib/portage/gpkg.py | 201 +-- lib/portage/tests/gpkg/test_gpkg_checksum.py | 21 +-- lib/portage/tests/gpkg/test_gpkg_gpg.py | 28 ++-- 5 files changed, 158 insertions(+), 102 deletions(-) diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py index ed2fda827..f3f149b14 100644 --- a/lib/portage/binpkg.py +++ b/lib/portage/binpkg.py @@ -17,8 +17,8 @@ def get_binpkg_format(binpkg_path): try: with open(binpkg_path, "rb") as binpkg_file: -header = binpkg_file.read(6) -if header == b"gpkg-1": +header = binpkg_file.read(100) +if b"/gpkg-1\x00" in header: file_format = "gpkg" else: binpkg_file.seek(-16, 2) @@ -32,7 +32,9 @@ def get_binpkg_format(binpkg_path): if file_format is None: try: with tarfile.open(binpkg_path) as gpkg_tar: -if "gpkg-1" in gpkg_tar.getnames(): +if "gpkg-1" in [ +f.split("/", maxsplit=1)[-1] for f in gpkg_tar.getnames() +]: file_format = "gpkg" except tarfile.TarError: pass diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 0857ff21a..814e6627c 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -744,7 +744,7 @@ class binarytree: mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) elif binpkg_format == "gpkg": mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, update_path) -mybinpkg.update_metadata(mydata, newcpv=mynewcpv) +mybinpkg.update_metadata(mydata, new_basename=mynewcpv) else: raise InvalidBinaryPackageFormat(binpkg_format) self.inject(mynewcpv, filename=update_path) diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py index 644ff412b..5f8e19341 100644 --- a/lib/portage/gpkg.py +++ b/lib/portage/gpkg.py @@ -764,10 +764,10 @@ class gpkg: https://www.gentoo.org/glep/glep-0078.html """ -def __init__(self, settings, base_name=None, gpkg_file=None): +def __init__(self, settings, basename=None, gpkg_file=None): """ gpkg class handle all gpkg operations for one package. -base_name is the package basename. +basename is the package basename. gpkg_file should be exists file path for read or will create. """ self.settings = settings @@ -778,10 +778,16 @@ class gpkg: self.gpkg_file = _unicode_decode( gpkg_file, encoding=_encodings["fs"], errors="strict" ) -self.base_name = base_name + +if basename is None: +self.basename = None +else: +self.basename = basename.split("/", maxsplit=1)[-1] + self.checksums = [] self.manifest_old = [] -signature_exist = None +self.signature_exist = None +self.prefix = None # Compression is the compression algorithm, if set to None will # not use compression. @@ -892,7 +898,9 @@ class gpkg: # Check gpkg and metadata with tarfile.open(mode="r", fileobj=container_file) as container: -if self.gpkg_version not in container.getnames(): +if self.gpkg_version not in ( +os.path.basename(f) for f in container.getnames() +): raise InvalidBinaryPackageFormat("Invalid gpkg file.") metadata_tarinfo, metadata_comp = self._get_inner_tarinfo( @@ -985,7 +993,7 @@ class gpkg: ) # Long CPV -if len(self.base_name) >= 154: +if len(self.basename) >= 154: container_tar_format = tarfile.GNU_FORMAT # gpkg container @@ -994,9 +1002,14 @@ class gpkg: ) # gpkg version -gpkg_version_file = tarfile.TarInfo(self.gpkg_version) +gpkg_version_file = tarfile.TarInfo( +os.path.join(self.basename, self.gpkg_version) +) gpkg_version_file.mtime = datetime.utcnow().timestamp() container.addfile(gpkg_version_file) +checksum_info = checksum_helper(self.settings) +checksum_info.finish() +self._record_checksum(checksum_info, gpkg_version_file)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 57ce385e32e79b9d332fe1fdb3be50bdb07e7838 Author: Mike Gilbert gentoo org> AuthorDate: Thu Aug 18 17:59:03 2022 + Commit: Mike Gilbert gentoo org> CommitDate: Thu Aug 18 18:11:25 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=57ce385e dbapi: avoid iterating porttrees twice in _set_porttrees() If porttrees is a generator object, the second pass will fail. Bug: https://bugs.gentoo.org/865635 Fixes: 9e24d0143450628f334cdb62e579efafd1bfd2ba Signed-off-by: Mike Gilbert gentoo.org> lib/portage/dbapi/porttree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index cd919ba31..126e7161b 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -367,14 +367,14 @@ class portdbapi(dbapi): repo priority @type porttrees: list """ +self._porttrees = tuple(porttrees) self._porttrees_repos = portage.OrderedDict( (repo.name, repo) for repo in ( self.repositories.get_repo_for_location(location) -for location in porttrees +for location in self._porttrees ) ) -self._porttrees = tuple(porttrees) def _get_porttrees(self): return self._porttrees
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: e893f4fc12eb618318b1945ce7a05a94fb1ea1b4 Author: Zac Medico gentoo org> AuthorDate: Sun Apr 17 19:12:40 2022 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 18 02:18:31 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e893f4fc MergeProcess: propagate mtimedb["ldpath"] to parent process (bug 836375) Use an instance of multiprocessing.Pipe to propagate mtimedb["ldpath"] from the MergeProcess child process to the parent process. This fixes env_update calls to avoid unnecessary regeneration of ld.so.cache in cases where mtimedb["ldpath"] has not changed since the last call to env_update. Bug: https://bugs.gentoo.org/836375 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/_MergeProcess.py | 22 ++ lib/portage/dbapi/vartree.py | 10 ++ 2 files changed, 32 insertions(+) diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index db3f3b105..197c48a7e 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 import io +import multiprocessing import platform import fcntl @@ -38,6 +39,7 @@ class MergeProcess(ForkProcess): "_dblink", "_elog_keys", "_locked_vdb", +"_mtime_reader", ) def _start(self): @@ -113,6 +115,19 @@ class MergeProcess(ForkProcess): self._elog_reader_fd = None return False +def _mtime_handler(self): +if self._mtime_reader is not None: +try: +mtimes = self._mtime_reader.recv() +except EOFError: +self.scheduler.remove_reader(self._mtime_reader.fileno()) +self._mtime_reader.close() +self._mtime_reader = None +else: +if self.prev_mtimes is not None: +self.prev_mtimes.clear() +self.prev_mtimes.update(mtimes) + def _spawn(self, args, fd_pipes, **kwargs): """ Extend the superclass _spawn method to perform some pre-fork and @@ -127,6 +142,11 @@ class MergeProcess(ForkProcess): fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK, ) +mtime_reader, mtime_writer = multiprocessing.Pipe(duplex=False) +fd_pipes[mtime_writer.fileno()] = mtime_writer.fileno() +self.scheduler.add_reader(mtime_reader.fileno(), self._mtime_handler) +self._mtime_reader = mtime_reader + blockers = None if self.blockers is not None: # Query blockers in the main process, since closing @@ -142,6 +162,7 @@ class MergeProcess(ForkProcess): vartree=self.vartree, blockers=blockers, pipe=elog_writer_fd, +mtime_pipe=mtime_writer, ) fd_pipes[elog_writer_fd] = elog_writer_fd self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler) @@ -160,6 +181,7 @@ class MergeProcess(ForkProcess): self._elog_reader_fd = elog_reader_fd pids = super(MergeProcess, self)._spawn(args, fd_pipes, **kwargs) os.close(elog_writer_fd) +mtime_writer.close() self._buf = "" self._elog_keys = set() # Discard messages which will be collected by the subprocess, diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 602913862..a95d60691 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -1806,6 +1806,7 @@ class dblink: blockers=None, scheduler=None, pipe=None, +mtime_pipe=None, ): """ Creates a DBlink object for a given CPV. @@ -1862,6 +1863,7 @@ class dblink: self._device_path_map = {} self._hardlink_merge_map = {} self._hash_key = (self._eroot, self.mycpv) +self._mtime_pipe = mtime_pipe self._protect_obj = None self._pipe = pipe self._postinst_failure = False @@ -2618,6 +2620,7 @@ class dblink: writemsg_level=self._display_merge, vardbapi=self.vartree.dbapi, ) +self._send_mtimes(ldpath_mtimes) unmerge_with_replacement = preserve_paths is not None if not unmerge_with_replacement: @@ -4243,6 +4246,12 @@ class dblink: def _emerge_log(self, msg): emergelog(False, msg) +def _send_mtimes(self, mtimes): +if self._mtime_pipe is None: +return + +self._mtime_pipe.send(mtimes) + def treewalk( self, srcroot, @@ -5274,6 +5283,7 @@ class dblink: writemsg_level=self._display_merge, vardbapi=self.vartree.dbapi, ) +self._send_mtimes(prev_mtimes) # For gcc upgrades, preserved libs have to be removed after the # the library path has been upd
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 05c0b3d03393fe376f3ae4b49bce403d496e3e68 Author: Mike Frysinger chromium org> AuthorDate: Tue Mar 29 07:12:25 2022 + Commit: Sam James gentoo org> CommitDate: Wed Apr 13 15:34:30 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=05c0b3d0 vartree: avoid lock contention when there are no blockers No sense in grabbing the vdb lock if we aren't going to do any work. This avoids contention on the global lock with parallel packages. [sam: cherry-picked from chromiumos' third_party/portage_tool repo] (cherry picked from commit ea5f6f8c0a5e05d7630f9070992a89fa6907cc14) Signed-off-by: Sam James gentoo.org> Closes: https://github.com/gentoo/portage/pull/813 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 19 +++ 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index e7252790d..602913862 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -5148,14 +5148,17 @@ class dblink: self._clear_contents_cache() contents = self.getcontents() destroot_len = len(destroot) - 1 -self.lockdb() -try: -for blocker in blockers: -self.vartree.dbapi.removeFromContents( -blocker, iter(contents), relative_paths=False -) -finally: -self.unlockdb() + +# Avoid lock contention if we aren't going to do any work. +if blockers: +self.lockdb() +try: +for blocker in blockers: +self.vartree.dbapi.removeFromContents( +blocker, iter(contents), relative_paths=False +) +finally: +self.unlockdb() plib_registry = self.vartree.dbapi._plib_registry if plib_registry:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 964c0e16172b76a8ebdd737cd8919870ae2b5f96 Author: Mike Frysinger chromium org> AuthorDate: Tue Mar 29 13:56:55 2022 + Commit: Sam James gentoo org> CommitDate: Wed Apr 13 15:34:30 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=964c0e16 vartree: skip env-update if no updates were merged This speeds up virtual/ installs by not constantly re-running env-update. [sam: cherry-picked from chromiumos' third_party/portage_tool repo] (cherry picked from commit 87ac3566ebb7155a57876d345849bd0fd6878c0e) Bug: https://bugs.gentoo.org/836375 Signed-off-by: Sam James gentoo.org> lib/portage/dbapi/vartree.py | 38 +- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 072bc1506..e7252790d 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -2608,14 +2608,16 @@ class dblink: else: self.settings.pop("PORTAGE_LOG_FILE", None) -env_update( -target_root=self.settings["ROOT"], -prev_mtimes=ldpath_mtimes, -contents=contents, -env=self.settings, -writemsg_level=self._display_merge, -vardbapi=self.vartree.dbapi, -) +# If we didn't unmerge anything, don't bother updating env. +if contents: +env_update( +target_root=self.settings["ROOT"], +prev_mtimes=ldpath_mtimes, +contents=contents, +env=self.settings, +writemsg_level=self._display_merge, +vardbapi=self.vartree.dbapi, +) unmerge_with_replacement = preserve_paths is not None if not unmerge_with_replacement: @@ -5258,15 +5260,17 @@ class dblink: ], ) -# update environment settings, library paths. DO NOT change symlinks. -env_update( -target_root=self.settings["ROOT"], -prev_mtimes=prev_mtimes, -contents=contents, -env=self.settings, -writemsg_level=self._display_merge, -vardbapi=self.vartree.dbapi, -) +# Update environment settings, library paths. DO NOT change symlinks. +# Only do this if we actually installed something. +if contents: +env_update( +target_root=self.settings["ROOT"], +prev_mtimes=prev_mtimes, +contents=contents, +env=self.settings, +writemsg_level=self._display_merge, +vardbapi=self.vartree.dbapi, +) # For gcc upgrades, preserved libs have to be removed after the # the library path has been updated.
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/_emerge/, cnf/
commit: eecd073d274c6d669f324cbb31151789d32f87fb Author: Sam James gentoo org> AuthorDate: Sat Mar 5 05:57:25 2022 + Commit: Sam James gentoo org> CommitDate: Wed Apr 13 03:05:45 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=eecd073d autoclean: only enable for global updates and depcleans Closes: https://bugs.gentoo.org/792195 Signed-off-by: Sam James gentoo.org> Closes: https://github.com/gentoo/portage/pull/795 Signed-off-by: Sam James gentoo.org> cnf/make.globals | 2 +- lib/_emerge/actions.py | 22 -- lib/portage/dbapi/vartree.py | 24 +--- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/cnf/make.globals b/cnf/make.globals index 69b365f71..f42cffe8b 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -101,7 +101,7 @@ EMERGE_WARNING_DELAY="10" # Automatically clean installed packages after they are updated. # This option will be removed and forced to yes. -AUTOCLEAN="yes" +AUTOCLEAN="no" PORTAGE_BZIP2_COMMAND="bzip2" diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index f82069f9a..e697459c8 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -672,12 +672,6 @@ def action_build( ldpath_mtimes, autoclean=1, ) -else: -portage.writemsg_stdout( -colorize("WARN", "WARNING:") -+ " AUTOCLEAN is disabled. This can cause serious" -+ " problems due to overlapping packages.\n" -) return retval @@ -776,6 +770,15 @@ def action_depclean( # relatively safe to ignore missing deps when only asked to remove # specific packages. +# Force autoclean for depcleans (but not purges), as it was changed +# to default off to not run it on every unmerge. +# bug #792195 +if action == "depclean": +settings.unlock() +settings["AUTOCLEAN"] = "yes" +settings.backup_changes("AUTOCLEAN") +settings.lock() + msg = [] if ( "preserve-libs" not in settings.features @@ -3386,6 +3389,13 @@ def run_action(emerge_config): # Reload the whole config from scratch. load_emerge_config(emerge_config=emerge_config) +# Let's autoclean if we applied updates, rather than always doing it +# bug #792195 +emerge_config.target_config.settings.unlock() +emerge_config.target_config.settings["AUTOCLEAN"] = "yes" +emerge_config.target_config.settings.backup_changes("AUTOCLEAN") +emerge_config.target_config.settings.lock() + xterm_titles = "notitles" not in emerge_config.target_config.settings.features if xterm_titles: xtermTitle("emerge") diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 863efe9cc..190bf8c92 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -5086,21 +5086,18 @@ class dblink: emerge_log = self._emerge_log -# If we have any preserved libraries then autoclean -# is forced so that preserve-libs logic doesn't have +# We always autoclean now for the current package-case for simplicity. +# If it were conditional, we'd always need to do it when any preserved-libs, +# so that preserve-libs logic doesn't have # to account for the additional complexity of the # AUTOCLEAN=no mode. -autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes" or preserve_paths - -if autoclean: -emerge_log(_(" >>> AUTOCLEAN: %s") % (slot_atom,)) +emerge_log(_(" >>> AUTOCLEAN: %s") % (slot_atom,)) others_in_slot.append(self) # self has just been merged for dblnk in list(others_in_slot): if dblnk is self: continue -if not (autoclean or dblnk.mycpv == self.mycpv or reinstall_self): -continue + showMessage(_(">>> Safely unmerging already-installed instance...\n")) emerge_log(_(" === Unmerging... (%s)") % (dblnk.mycpv,)) others_in_slot.remove(dblnk) # dblnk will unmerge itself now @@ -5130,17 +5127,6 @@ class dblink: self.unlockdb() showMessage(_(">>> Original instance of package unmerged safely.\n")) -if len(others_in_slot) > 1: -showMessage( -colorize("WARN", _("WARNING:")) -+ _( -" AUTOCLEAN is disabled. This can cause serious" -" problems due to overlapping packages.\n" -), -level=logging.WARN, -noiselevel=-1, -) - # We hold both directory locks. self.dbdir = self.dbpkgdir self.lockdb()
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 0009404ee5f293fe7076d59a52c46827c3b77738 Author: Mike Gilbert gentoo org> AuthorDate: Fri Nov 26 17:43:45 2021 + Commit: Mike Gilbert gentoo org> CommitDate: Fri Nov 26 17:43:45 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0009404e dbapi: fix logic in bintree.binarytree._parse_build_id() Resolves an error when improperly named xpak files exist in PKGDIR. Bug: https://bugs.gentoo.org/818886 Signed-off-by: Mike Gilbert gentoo.org> lib/portage/dbapi/bintree.py | 9 - 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 777fc4918..9dbf9ee8b 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1877,11 +1877,10 @@ class binarytree: suffixlen = len(".xpak") hyphen = filename.rfind("-", 0, -(suffixlen + 1)) if hyphen != -1: -build_id = filename[hyphen + 1 : -suffixlen] -try: -build_id = int(build_id) -except ValueError: -pass +try: +build_id = int(filename[hyphen + 1 : -suffixlen]) +except ValueError: +pass return build_id def isremote(self, pkgname):
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 95935004862d1363a0e3d5b6ee33d4ed65566a2d Author: Zac Medico gentoo org> AuthorDate: Tue Sep 21 05:13:12 2021 + Commit: Zac Medico gentoo org> CommitDate: Tue Sep 21 05:33:51 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=95935004 portdbapi: convert compat coroutine to async Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/porttree.py | 10 -- 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 72b875879..93f3fee2f 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -38,7 +38,6 @@ from portage import os from portage import _encodings from portage import _unicode_encode from portage.util.futures import asyncio -from portage.util.futures.compat_coroutine import coroutine, coroutine_return from portage.util.futures.iter_completed import iter_gather from _emerge.EbuildMetadataPhase import EbuildMetadataPhase @@ -1244,8 +1243,7 @@ class portdbapi(dbapi): loop = self._event_loop return loop.run_until_complete(self.async_xmatch(level, origdep, loop=loop)) -@coroutine -def async_xmatch(self, level, origdep, loop=None): +async def async_xmatch(self, level, origdep, loop=None): """ Asynchronous form of xmatch. @@ -1269,7 +1267,7 @@ class portdbapi(dbapi): if self.frozen: cache_key = (mydep, mydep.unevaluated_atom) try: -coroutine_return(self.xcache[level][cache_key][:]) +return self.xcache[level][cache_key][:] except KeyError: pass @@ -1336,7 +1334,7 @@ class portdbapi(dbapi): zip( aux_keys, ( -yield self.async_aux_get( +await self.async_aux_get( cpv, aux_keys, myrepo=cpv.repo, loop=loop ) ), @@ -1384,7 +1382,7 @@ class portdbapi(dbapi): if not isinstance(myval, _pkg_str): myval = myval[:] -coroutine_return(myval) +return myval def match(self, mydep, use_cache=1): return self.xmatch("match-visible", mydep)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: bcda30d0a6fa4962cc7597dbed9b648cf6400ab5 Author: Zac Medico gentoo org> AuthorDate: Tue Sep 21 05:21:55 2021 + Commit: Zac Medico gentoo org> CommitDate: Tue Sep 21 05:33:51 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bcda30d0 vardbapi: convert compat coroutine to async Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 15 ++- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 7f3b5d773..8ffb23b1c 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -78,7 +78,6 @@ from portage import _os_merge from portage import _selinux_merge from portage import _unicode_decode from portage import _unicode_encode -from portage.util.futures.compat_coroutine import coroutine from portage.util.futures.executor.fork import ForkExecutor from ._VdbMetadataDelta import VdbMetadataDelta @@ -1006,8 +1005,7 @@ class vardbapi(dbapi): pass self._bump_mtime(cpv) -@coroutine -def unpack_metadata(self, pkg, dest_dir, loop=None): +async def unpack_metadata(self, pkg, dest_dir, loop=None): """ Unpack package metadata to a directory. This method is a coroutine. @@ -1029,10 +1027,9 @@ class vardbapi(dbapi): shutil.copy(os.path.join(parent, key), os.path.join(dest_dir, key)) break -yield loop.run_in_executor(ForkExecutor(loop=loop), async_copy) +await loop.run_in_executor(ForkExecutor(loop=loop), async_copy) -@coroutine -def unpack_contents( +async def unpack_contents( self, pkg, dest_dir, @@ -1097,10 +1094,10 @@ class vardbapi(dbapi): tar_cmd = ("tar", "-x", "--xattrs", "--xattrs-include=*", "-C", dest_dir) pr, pw = os.pipe() -proc = yield asyncio.create_subprocess_exec(*tar_cmd, stdin=pr) +proc = await asyncio.create_subprocess_exec(*tar_cmd, stdin=pr) os.close(pr) with os.fdopen(pw, "wb", 0) as pw_file: -excluded_config_files = yield loop.run_in_executor( +excluded_config_files = await loop.run_in_executor( ForkExecutor(loop=loop), functools.partial( self._dblink(cpv).quickpkg, @@ -1109,7 +1106,7 @@ class vardbapi(dbapi): include_unmodified_config=opts.include_unmodified_config == "y", ), ) -yield proc.wait() +await proc.wait() if proc.returncode != os.EX_OK: raise PortageException("command failed: {}".format(tar_cmd))
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 7957a7f4c2ef4656f6f9a29d2c9bab6e44daae1f Author: Zac Medico gentoo org> AuthorDate: Tue Sep 21 05:08:21 2021 + Commit: Zac Medico gentoo org> CommitDate: Tue Sep 21 05:33:51 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7957a7f4 bindbapi: convert compat coroutine to async Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 15 ++- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 024a9b5f6..777fc4918 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -39,7 +39,6 @@ from portage.localization import _ from portage.package.ebuild.profile_iuse import iter_iuse_vars from portage.util.file_copy import copyfile from portage.util.futures import asyncio -from portage.util.futures.compat_coroutine import coroutine from portage.util.futures.executor.fork import ForkExecutor from portage import _movefile from portage import os @@ -253,8 +252,7 @@ class bindbapi(fakedbapi): # inject will clear stale caches via cpv_inject. self.bintree.inject(cpv) -@coroutine -def unpack_metadata(self, pkg, dest_dir, loop=None): +async def unpack_metadata(self, pkg, dest_dir, loop=None): """ Unpack package metadata to a directory. This method is a coroutine. @@ -271,17 +269,16 @@ class bindbapi(fakedbapi): key = self._instance_key(cpv) add_pkg = self.bintree._additional_pkgs.get(key) if add_pkg is not None: -yield add_pkg._db.unpack_metadata(pkg, dest_dir, loop=loop) +await add_pkg._db.unpack_metadata(pkg, dest_dir, loop=loop) else: tbz2_file = self.bintree.getname(cpv) -yield loop.run_in_executor( +await loop.run_in_executor( ForkExecutor(loop=loop), portage.xpak.tbz2(tbz2_file).unpackinfo, dest_dir, ) -@coroutine -def unpack_contents(self, pkg, dest_dir, loop=None): +async def unpack_contents(self, pkg, dest_dir, loop=None): """ Unpack package contents to a directory. This method is a coroutine. @@ -313,7 +310,7 @@ class bindbapi(fakedbapi): ) extractor.start() -yield extractor.async_wait() +await extractor.async_wait() if extractor.returncode != os.EX_OK: raise PortageException("Error Extracting '{}'".format(pkg_path)) @@ -322,7 +319,7 @@ class bindbapi(fakedbapi): add_pkg = self.bintree._additional_pkgs.get(instance_key) if add_pkg is None: raise portage.exception.PackageNotFound(cpv) -yield add_pkg._db.unpack_contents(pkg, dest_dir, loop=loop) +await add_pkg._db.unpack_contents(pkg, dest_dir, loop=loop) def cp_list(self, *pargs, **kwargs): if not self.bintree.populated:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 2fe8238b540240a62de3513839974e033f0bacd2 Author: Daniel M. Weeks danweeks net> AuthorDate: Mon May 31 16:10:21 2021 + Commit: Zac Medico gentoo org> CommitDate: Sat Jun 5 18:03:45 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2fe8238b Support GLEP 75 in PORTAGE_RO_DISTDIRS size check Closes: https://github.com/gentoo/portage/pull/724 Signed-off-by: Daniel M. Weeks danweeks.net> Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/porttree.py | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 99c36450c..0f50f1338 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -12,7 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.dbapi.dep_expand:dep_expand', 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot', 'portage.package.ebuild.doebuild:doebuild', - 'portage.package.ebuild.fetch:_download_suffix', + 'portage.package.ebuild.fetch:get_mirror_url,_download_suffix', 'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level', 'portage.util.listdir:listdir', 'portage.versions:best,catsplit,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str', @@ -859,7 +859,11 @@ class portdbapi(dbapi): if ro_distdirs is not None: for x in shlex_split(ro_distdirs): try: - mystat = os.stat(os.path.join(x, myfile)) + mystat = os.stat( + portage.package.ebuild.fetch.get_mirror_url( + x, myfile, self.settings + ) + ) except OSError: pass else:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/_emerge/
commit: 8879cef9006d2277453aaee407c234a2d1bc47ba Author: Florian Schmaus geekplace eu> AuthorDate: Tue Mar 9 07:25:59 2021 + Commit: Zac Medico gentoo org> CommitDate: Thu Mar 11 12:04:41 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8879cef9 Use atomic_ofstream as Context Manager i.e., with-statement contexts With [1: e93e6d65fa1c] atomic_ofstream became a Context Manager. This commit transforms three further call sites of atomic_ofstream() to use with-statement contexts for easier readability and increased robustness against resource leaks. 1: e93e6d65fa1ca75f676a227f7918f8b6d747425c Make atomic_ofstream a Context Manager Signed-off-by: Florian Schmaus geekplace.eu> Signed-off-by: Zac Medico gentoo.org> lib/_emerge/BlockerCache.py| 6 +++--- lib/portage/dbapi/_VdbMetadataDelta.py | 11 +-- lib/portage/dbapi/vartree.py | 6 +++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/_emerge/BlockerCache.py b/lib/_emerge/BlockerCache.py index 8154d9ade..035f2212d 100644 --- a/lib/_emerge/BlockerCache.py +++ b/lib/_emerge/BlockerCache.py @@ -133,9 +133,9 @@ class BlockerCache(portage.cache.mappings.MutableMapping): if len(self._modified) >= self._cache_threshold and \ secpass >= 2: try: - f = portage.util.atomic_ofstream(self._cache_filename, mode='wb') - pickle.dump(self._cache_data, f, protocol=2) - f.close() + with portage.util.atomic_ofstream(self._cache_filename, mode='wb') as f: + pickle.dump(self._cache_data, f, protocol=2) + portage.util.apply_secpass_permissions( self._cache_filename, gid=portage.portage_gid, mode=0o644) except (IOError, OSError): diff --git a/lib/portage/dbapi/_VdbMetadataDelta.py b/lib/portage/dbapi/_VdbMetadataDelta.py index ffdc0b361..568e1964a 100644 --- a/lib/portage/dbapi/_VdbMetadataDelta.py +++ b/lib/portage/dbapi/_VdbMetadataDelta.py @@ -18,13 +18,12 @@ class VdbMetadataDelta: self._vardb = vardb def initialize(self, timestamp): - f = atomic_ofstream(self._vardb._cache_delta_filename, 'w', - encoding=_encodings['repo.content'], errors='strict') - json.dump({ - "version": self._format_version, - "timestamp": timestamp + with atomic_ofstream(self._vardb._cache_delta_filename, 'w', + encoding=_encodings['repo.content'], errors='strict') as f: + json.dump({ + "version": self._format_version, + "timestamp": timestamp }, f, ensure_ascii=False) - f.close() def load(self): diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 826083eae..5ae035baf 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -614,9 +614,9 @@ class vardbapi(dbapi): timestamp = time.time() self._aux_cache["timestamp"] = timestamp - f = atomic_ofstream(self._aux_cache_filename, 'wb') - pickle.dump(self._aux_cache, f, protocol=2) - f.close() + with atomic_ofstream(self._aux_cache_filename, 'wb') as f: + pickle.dump(self._aux_cache, f, protocol=2) + apply_secpass_permissions( self._aux_cache_filename, mode=0o644)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 6eafeb89fb4d17326d6ffa1d6ca83e5407890d42 Author: Zac Medico gentoo org> AuthorDate: Sun Mar 7 09:03:33 2021 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 7 09:30:27 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6eafeb89 dblink: add _format_contents_line method Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 42 +- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 2c75be4a1..826083eae 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -5045,7 +5045,14 @@ class dblink: % (relative_path, myabsto)]) showMessage("%s %s -> %s\n" % (zing, mydest, myto)) - outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 10)+"\n") + outfile.write( + self._format_contents_line( + node_type="sym", + abs_path=myrealdest, + symlink_target=myto, + mtime_ns=mymtime, + ) + ) else: showMessage(_("!!! Failed to move file.\n"), level=logging.ERROR, noiselevel=-1) @@ -5146,7 +5153,9 @@ class dblink: except OSError: pass - outfile.write("dir "+myrealdest+"\n") + outfile.write( + self._format_contents_line(node_type="dir", abs_path=myrealdest) + ) # recurse and merge this directory mergelist.extend(join(relative_path, child) for child in os.listdir(join(srcroot, relative_path))) @@ -5194,7 +5203,14 @@ class dblink: pass if mymtime != None: - outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime // 10)+"\n") + outfile.write( + self._format_contents_line( + node_type="obj", + abs_path=myrealdest, + md5_digest=mymd5, + mtime_ns=mymtime, + ) + ) showMessage("%s %s\n" % (zing,mydest)) else: # we are merging a fifo or device node @@ -5214,9 +5230,13 @@ class dblink: else: return 1 if stat.S_ISFIFO(mymode): - outfile.write("fif %s\n" % myrealdest) + outfile.write( + self._format_contents_line(node_type="fif", abs_path=myrealdest) + ) else: - outfile.write("dev %s\n" % myrealdest) + outfile.write( + self._format_contents_line(node_type="dev", abs_path=myrealdest) + ) showMessage(zing + " " + mydest + "\n") def _protect(self, cfgfiledict, protect_if_modified, src_md5, @@ -5278,6 +5298,18 @@ class dblink: return dest, protected, move_me + def _format_contents_line( + self, node_type, abs_path, md5_digest=None, symlink_target=None, mtime_ns=None + ): + fields = [node_type, abs_path] + if md5_digest is not None: + fields.append(md5_digest) + elif symlink_target is not None: + fields.append("-> {}".format(symlink_target)) + if mtime_ns is not None: + fields.append(str(mtime_ns // 10)) + return "{}\n".format(" ".join(fields)) + def _merged_path(self, path, lstatobj, ex
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: a2c59fa326ecacf9834df574a81f4a5e5b7d6b93 Author: Zac Medico gentoo org> AuthorDate: Tue Feb 23 21:29:57 2021 + Commit: Zac Medico gentoo org> CommitDate: Tue Feb 23 21:30:19 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a2c59fa3 binarytree.inject: create PKGDIR Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index ab09b42bc..429c8e64a 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1255,6 +1255,7 @@ class binarytree: # process) and then updated it, all while holding a lock. pkgindex_lock = None try: + os.makedirs(self.pkgdir, exist_ok=True) pkgindex_lock = lockfile(self._pkgindex_file, wantnewlockfile=1) if filename is not None:
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 7007dbcd1013829466498d7e0708c8b84fdd68bf Author: Zac Medico gentoo org> AuthorDate: Mon Jan 18 08:45:34 2021 + Commit: Zac Medico gentoo org> CommitDate: Mon Jan 18 08:46:44 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7007dbcd bindbapi.aux_update: fix spurious package file renames Omit the filename argument for the binarytree.inject method in aux_update, since an in-place update is desired, and the filename argument causes the file to be renamed when binpkg-multi-instance is enabled. Fixes: 328dd4712f88 ("binpkg-multi-instance 3 of 7") Bug: https://bugs.gentoo.org/765847 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index e5ba5893e..180e48c3b 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -214,7 +214,7 @@ class bindbapi(fakedbapi): del mydata[k] mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) # inject will clear stale caches via cpv_inject. - self.bintree.inject(cpv, filename=tbz2path) + self.bintree.inject(cpv) @coroutine
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 3d39e6891a3ee002fb1aa039c5660e6015c555d2 Author: Zac Medico gentoo org> AuthorDate: Sun Jan 17 13:24:21 2021 + Commit: Zac Medico gentoo org> CommitDate: Sun Jan 17 13:27:25 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d39e689 bindbapi.move_ent: use cpv_exists instead of getname The getname method is complicated by binpkg-multi-instance and soon BINPKG_FORMAT, so it's much simpler to use cpv_exists. Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 528a68979..e5ba5893e 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -483,7 +483,7 @@ class binarytree: myoldpkg = catsplit(mycpv)[1] mynewpkg = catsplit(mynewcpv)[1] - if (mynewpkg != myoldpkg) and os.path.exists(self.getname(mynewcpv)): + if (mynewpkg != myoldpkg) and self.dbapi.cpv_exists(mynewcpv): writemsg(_("!!! Cannot update binary: Destination exists.\n"), noiselevel=-1) writemsg("!!! "+mycpv+" -> "+mynewcpv+"\n", noiselevel=-1)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 375eb5c02df19833ddeb2c63438aac9faf3d5eae Author: Zac Medico gentoo org> AuthorDate: Sun Jan 17 08:35:41 2021 + Commit: Zac Medico gentoo org> CommitDate: Sun Jan 17 08:46:23 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=375eb5c0 vardbapi.move_ent: remove redundant self._pkg_str calls Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 8 ++-- 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index f3d74cf82..2c75be4a1 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2020 Gentoo Authors +# Copyright 1998-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -372,11 +372,7 @@ class vardbapi(dbapi): if not origmatches: return moves for mycpv in origmatches: - try: - mycpv = self._pkg_str(mycpv, None) - except (KeyError, InvalidData): - continue - mycpv_cp = cpv_getkey(mycpv) + mycpv_cp = mycpv.cp if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: bdc75b3aa217649f6d835b04eb3cac879459c0a0 Author: Zac Medico gentoo org> AuthorDate: Sun Jan 17 08:47:13 2021 + Commit: Zac Medico gentoo org> CommitDate: Sun Jan 17 08:47:26 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bdc75b3a bindbapi.move_ent: remove redundant self._pkg_str calls Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 10 +++--- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 7e24589e5..528a68979 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2020 Gentoo Authors +# Copyright 1998-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["bindbapi", "binarytree"] @@ -27,7 +27,7 @@ from portage.cache.mappings import slot_dict_class from portage.const import BINREPOS_CONF_FILE, CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS from portage.dbapi.virtual import fakedbapi from portage.dep import Atom, use_reduce, paren_enclose -from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \ +from portage.exception import AlarmSignal, InvalidPackageName, \ ParseError, PortageException from portage.localization import _ from portage.package.ebuild.profile_iuse import iter_iuse_vars @@ -466,11 +466,7 @@ class binarytree: if not origmatches: return moves for mycpv in origmatches: - try: - mycpv = self.dbapi._pkg_str(mycpv, None) - except (KeyError, InvalidData): - continue - mycpv_cp = portage.cpv_getkey(mycpv) + mycpv_cp = mycpv.cp if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/package/ebuild/
commit: 87ca3d4b09324a2359ea10aa69c1f2875c531486 Author: Zac Medico gentoo org> AuthorDate: Mon Dec 7 02:54:26 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Dec 7 02:54:52 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=87ca3d4b Remove unused EventLoop lazy imports Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 1 - lib/portage/package/ebuild/doebuild.py | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 1547d2f6d..f3d74cf82 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -41,7 +41,6 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap', 'portage.util._dyn_libs.NeededEntry:NeededEntry', 'portage.util._async.SchedulerInterface:SchedulerInterface', - 'portage.util._eventloop.EventLoop:EventLoop', 'portage.util._eventloop.global_event_loop:global_event_loop', 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,' + \ '_get_slot_re,_pkgsplit@pkgsplit,_pkg_str,_unknown_repo', diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py index 3b1991b28..f6cee4518 100644 --- a/lib/portage/package/ebuild/doebuild.py +++ b/lib/portage/package/ebuild/doebuild.py @@ -39,7 +39,6 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util._dyn_libs.NeededEntry:NeededEntry', 'portage.util._dyn_libs.soname_deps:SonameDepsProcessor', 'portage.util._async.SchedulerInterface:SchedulerInterface', - 'portage.util._eventloop.EventLoop:EventLoop', 'portage.util._eventloop.global_event_loop:global_event_loop', 'portage.util.ExtractKernelVersion:ExtractKernelVersion' )
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 6a3bdcc427c58112075c36cc0481c53215f12db4 Author: Zac Medico gentoo org> AuthorDate: Tue Sep 8 02:43:23 2020 + Commit: Zac Medico gentoo org> CommitDate: Tue Sep 8 02:44:46 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6a3bdcc4 binarytree.move_ent: fix path comparison prior to _movefile Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index e4393e06d..7e24589e5 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -531,7 +531,7 @@ class binarytree: new_path = self.getname(mynewcpv) self._pkg_paths[ self.dbapi._instance_key(mynewcpv)] = new_path[len(self.pkgdir)+1:] - if new_path != mytbz2: + if new_path != tbz2path: self._ensure_dir(os.path.dirname(new_path)) _movefile(tbz2path, new_path, mysettings=self.settings) self.inject(mynewcpv)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 656f8a7fcd2014c833e42282744c70a21e6c7e31 Author: Zac Medico gentoo org> AuthorDate: Sat Aug 8 23:51:52 2020 + Commit: Zac Medico gentoo org> CommitDate: Sat Aug 8 23:55:19 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=656f8a7f treewalk: reset config for unmerge (bug 711174) When cloning config instances for unmerge, call the reset method in order to ensure that there is no unintended leakage of variables which should not be shared. This fixes leakage of the PORTAGE_LOG_FILE variable, which triggered log corruption for FEATURES=compress-build-logs. Bug: https://bugs.gentoo.org/711174 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 5799d94f2..3eee025ad 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -4013,7 +4013,9 @@ class dblink: # since we need it to have private ${T} etc... for things # like elog. settings_clone = portage.config(clone=self.settings) - settings_clone.pop("PORTAGE_BUILDDIR_LOCKED", None) + # This reset ensures that there is no unintended leakage + # of variables which should not be shared. + settings_clone.reset() settings_clone.setcpv(cur_cpv, mydb=self.vartree.dbapi) if self._preserve_libs and "preserve-libs" in \ settings_clone["PORTAGE_RESTRICT"].split():
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: f74cae3d377006af710f4b70cfbbf3391a126bb5 Author: Aaron Bauman gentoo org> AuthorDate: Tue Aug 4 02:44:56 2020 + Commit: Zac Medico gentoo org> CommitDate: Tue Aug 4 03:13:01 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f74cae3d lib/portage/dbapi/vartree.py: fix reimported Signed-off-by: Aaron Bauman gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index fcf164e82..fbf455363 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -1619,7 +1619,6 @@ class dblink: At present this is implemented as a text backend in /var/db/pkg. """ - import re _normalize_needed = re.compile(r'//|^[^/]|./$|(^|/)\.\.?(/|$)') _contents_re = re.compile(r'^(' + \
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: ca6e92e3a08d0aad1dc551cfb1e380b843fc33b4 Author: Aaron Bauman gentoo org> AuthorDate: Mon Aug 3 22:43:25 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Aug 3 23:28:04 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ca6e92e3 lib/portage/dbapi/virtual.py: fix whitespace Signed-off-by: Aaron Bauman gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/virtual.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/virtual.py b/lib/portage/dbapi/virtual.py index f62fc2a30..4ed0dea1b 100644 --- a/lib/portage/dbapi/virtual.py +++ b/lib/portage/dbapi/virtual.py @@ -1,4 +1,4 @@ -# Copyright 1998-2013 Gentoo Foundation +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from portage.dbapi import dbapi @@ -214,7 +214,7 @@ class fakedbapi(dbapi): class testdbapi: """A dbapi instance with completely fake functions to get by hitting disk - TODO(antarus): + TODO(antarus): This class really needs to be rewritten to have better stubs; but these work for now. The dbapi classes themselves need unit tests...and that will be a lot of work. """
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 8af9f6d49363a1faeaa29a4fe00426dd2a1fabb4 Author: Aaron Bauman gentoo org> AuthorDate: Mon Aug 3 22:43:14 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Aug 3 23:28:02 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8af9f6d4 lib/portage/dbapi/bintree.py: fix whitespace Signed-off-by: Aaron Bauman gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 97be51121..59c265688 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1686,7 +1686,7 @@ class binarytree: resume = True writemsg(_("Resuming download of this tbz2, but it is possible that it is corrupt.\n"), noiselevel=-1) - + mydest = os.path.dirname(self.getname(pkgname)) self._ensure_dir(mydest) # urljoin doesn't work correctly with unrecognized protocols like sftp
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 3551cfb0be232df7f1a8d575181f8629ac7b4438 Author: Aaron Bauman gentoo org> AuthorDate: Mon Aug 3 20:20:25 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Aug 3 21:25:52 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3551cfb0 lib/portage/dbapi/IndexedPortdb.py: drop unused-import * Drop unused-import * Update copyright Signed-off-by: Aaron Bauman gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/IndexedPortdb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/portage/dbapi/IndexedPortdb.py b/lib/portage/dbapi/IndexedPortdb.py index 5f1cb5bd1..8d9ba89ff 100644 --- a/lib/portage/dbapi/IndexedPortdb.py +++ b/lib/portage/dbapi/IndexedPortdb.py @@ -1,4 +1,4 @@ -# Copyright 2014 Gentoo Foundation +# Copyright 2014-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -15,7 +15,6 @@ from portage.cache.index.IndexStreamIterator import IndexStreamIterator from portage.cache.index.pkg_desc_index import \ pkg_desc_index_line_read, pkg_desc_index_node from portage.util.iterators.MultiIterGroupBy import MultiIterGroupBy -from portage.versions import _pkg_str class IndexedPortdb: """
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 2af196d23ef2472432f0d8f29de9e4149978f8ef Author: Aaron Bauman gentoo org> AuthorDate: Mon Aug 3 20:20:22 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Aug 3 21:15:55 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2af196d2 lib/portage/dbapi/cpv_expand.py: drop unused-import * Drop unused import * Update copyright Signed-off-by: Aaron Bauman gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/cpv_expand.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/portage/dbapi/cpv_expand.py b/lib/portage/dbapi/cpv_expand.py index a1a91f554..eeec446d4 100644 --- a/lib/portage/dbapi/cpv_expand.py +++ b/lib/portage/dbapi/cpv_expand.py @@ -1,12 +1,10 @@ -# Copyright 2010-2013 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["cpv_expand"] import portage from portage.exception import AmbiguousPackageName -from portage.localization import _ -from portage.util import writemsg from portage.versions import _pkgsplit def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: c14bbdf7a32eca5d559c012682b6101d91d71bf2 Author: Aaron Bauman gentoo org> AuthorDate: Mon Aug 3 19:05:59 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Aug 3 19:22:24 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c14bbdf7 lib/portage/dbapi/bintree.py: drop unused-import Signed-off-by: Aaron Bauman gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/bintree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index e86fa5caa..97be51121 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -27,7 +27,7 @@ from portage.const import CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS from portage.dbapi.virtual import fakedbapi from portage.dep import Atom, use_reduce, paren_enclose from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \ - ParseError, PermissionDenied, PortageException + ParseError, PortageException from portage.localization import _ from portage.package.ebuild.profile_iuse import iter_iuse_vars from portage.util.futures import asyncio
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 309f2cc59e19eab05dfc2e86d1df4fc7ba1be93d Author: Zac Medico gentoo org> AuthorDate: Wed Jul 22 20:10:30 2020 + Commit: Zac Medico gentoo org> CommitDate: Wed Jul 22 20:12:16 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=309f2cc5 MergeProcess: handle cancelled future in _proc_join_done If the future was cancelled, then avoid a possible ValueError when accessing proc.exitcode. Fixes: f587ebf3d492 ("MergeProcess: handle RETURNCODE_POSTINST_FAILURE in _proc_join_done") Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/_MergeProcess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index 85deedc18..e89b53555 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -199,7 +199,7 @@ class MergeProcess(ForkProcess): """ Extend _proc_join_done to react to RETURNCODE_POSTINST_FAILURE. """ - if proc.exitcode == portage.const.RETURNCODE_POSTINST_FAILURE: + if not future.cancelled() and proc.exitcode == portage.const.RETURNCODE_POSTINST_FAILURE: self.postinst_failure = True self.returncode = os.EX_OK super(MergeProcess, self)._proc_join_done(proc, future)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: f587ebf3d4920b6c3cae3654918ba9cda6625a3e Author: Zac Medico gentoo org> AuthorDate: Wed Jul 22 19:32:38 2020 + Commit: Zac Medico gentoo org> CommitDate: Wed Jul 22 19:37:33 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f587ebf3 MergeProcess: handle RETURNCODE_POSTINST_FAILURE in _proc_join_done Since ForkProcess now receives process exit status in the _proc_join_done method instead of the _async_waitpid_cb method, MergeProcess needs to handle RETURNCODE_POSTINST_FAILURE there instead. Fixes: 3561071e07ad ("MergeProcess: replace os.fork with multiprocessing.Process (bug 730192)") Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/_MergeProcess.py | 9 - 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index 6924c8b0e..85deedc18 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -195,15 +195,14 @@ class MergeProcess(ForkProcess): prev_mtimes=self.prev_mtimes, counter=counter) return rval - def _async_waitpid_cb(self, *args, **kwargs): + def _proc_join_done(self, proc, future): """ - Override _async_waitpid_cb to perform cleanup that is - not necessarily idempotent. + Extend _proc_join_done to react to RETURNCODE_POSTINST_FAILURE. """ - ForkProcess._async_waitpid_cb(self, *args, **kwargs) - if self.returncode == portage.const.RETURNCODE_POSTINST_FAILURE: + if proc.exitcode == portage.const.RETURNCODE_POSTINST_FAILURE: self.postinst_failure = True self.returncode = os.EX_OK + super(MergeProcess, self)._proc_join_done(proc, future) def _unregister(self): """
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 3561071e07ad47db91bf0f2c2c2b02e2061b217c Author: Zac Medico gentoo org> AuthorDate: Sun Jul 19 06:31:37 2020 + Commit: Zac Medico gentoo org> CommitDate: Wed Jul 22 16:55:33 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3561071e MergeProcess: replace os.fork with multiprocessing.Process (bug 730192) Fix the MergeProcess _spawn method to call the superclass _spawn method, in order to replace os.fork with multiprocessing.Process, promoting a healthy state for the forked interpreter. Bug: https://bugs.gentoo.org/730192 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/_MergeProcess.py | 106 +++-- 1 file changed, 30 insertions(+), 76 deletions(-) diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index 274ef586f..6924c8b0e 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -3,9 +3,6 @@ import io import platform -import signal -import sys -import traceback import fcntl import portage @@ -24,7 +21,7 @@ class MergeProcess(ForkProcess): 'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild', 'mydbapi', 'postinst_failure', 'prev_mtimes', 'unmerge', '_elog_reader_fd', - '_buf', '_elog_keys', '_locked_vdb') + '_buf', '_counter', '_dblink', '_elog_keys', '_locked_vdb') def _start(self): # Portage should always call setcpv prior to this @@ -103,8 +100,8 @@ class MergeProcess(ForkProcess): def _spawn(self, args, fd_pipes, **kwargs): """ - Fork a subprocess, apply local settings, and call - dblink.merge(). TODO: Share code with ForkProcess. + Extend the superclass _spawn method to perform some pre-fork and + post-fork actions. """ elog_reader_fd, elog_writer_fd = os.pipe() @@ -132,57 +129,31 @@ class MergeProcess(ForkProcess): # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() - counter = None if not self.unmerge: - counter = self.vartree.dbapi.counter_tick() - - parent_pid = os.getpid() - pid = None - try: - pid = os.fork() - - if pid != 0: - if not isinstance(pid, int): - raise AssertionError( - "fork returned non-integer: %s" % (repr(pid),)) - - os.close(elog_writer_fd) - self._elog_reader_fd = elog_reader_fd - self._buf = "" - self._elog_keys = set() - # Discard messages which will be collected by the subprocess, - # in order to avoid duplicates (bug #446136). - portage.elog.messages.collect_messages(key=mylink.mycpv) - - # invalidate relevant vardbapi caches - if self.vartree.dbapi._categories is not None: - self.vartree.dbapi._categories = None - self.vartree.dbapi._pkgs_changed = True - self.vartree.dbapi._clear_pkg_cache(mylink) - - return [pid] - - os.close(elog_reader_fd) - - # Use default signal handlers in order to avoid problems - # killing subprocesses as reported in bug #353239. - signal.signal(signal.SIGINT, signal.SIG_DFL) - signal.signal(signal.SIGTERM, signal.SIG_DFL) - - # Unregister SIGCHLD handler and wakeup_fd for the parent - # process's event loop (bug 655656). - signal.signal(signal.SIGCHLD, signal.SIG_DFL) - try: - wakeup_fd = signal.set_wakeup_fd(-1) - if wakeup_fd > 0: - os.close(wakeup_fd) - except (ValueError, OSError): - pass - - portage.locks._close_fds() - # We don't exec, so use close_fds=False - # (see _setup_pipes docstring). - portage.process._setup_pipes(fd_pipes, close_fds=False) + self._counter = self.vartree.dbapi.counter_tick() + + self._dblink = mylink + self._elog_reader_fd = elog_reader_fd + pids = super(MergeProces
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, bin/, lib/portage/_emirrordist/, lib/portage/util/_dyn_libs/, ...
commit: 55a6bfdd9d3e2b8d0a59f5579de962fb82dde8ca Author: Michał Górny gentoo org> AuthorDate: Fri Jul 17 04:34:00 2020 + Commit: Michał Górny gentoo org> CommitDate: Fri Jul 17 04:39:07 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=55a6bfdd Remove support code for Python < 3.3 Signed-off-by: Michał Górny gentoo.org> bin/chpathtool.py | 8 +-- bin/doins.py | 14 + lib/portage/_emirrordist/FetchTask.py | 13 ++--- lib/portage/dbapi/vartree.py | 25 ++--- lib/portage/util/_compare_files.py | 22 +++- .../util/_dyn_libs/PreservedLibsRegistry.py| 7 +-- lib/portage/util/movefile.py | 60 ++ lib/portage/util/mtimedb.py| 7 +-- 8 files changed, 39 insertions(+), 117 deletions(-) diff --git a/bin/chpathtool.py b/bin/chpathtool.py index fbd18b987..c036046ae 100755 --- a/bin/chpathtool.py +++ b/bin/chpathtool.py @@ -128,12 +128,8 @@ def chpath_inplace(filename, is_text_file, old, new): f.close() if modified: - if sys.hexversion >= 0x303: - orig_mtime = orig_stat.st_mtime_ns - os.utime(filename, ns=(orig_mtime, orig_mtime)) - else: - orig_mtime = orig_stat[stat.ST_MTIME] - os.utime(filename, (orig_mtime, orig_mtime)) + orig_mtime = orig_stat.st_mtime_ns + os.utime(filename, ns=(orig_mtime, orig_mtime)) return modified def chpath_inplace_symlink(filename, st, old, new): diff --git a/bin/doins.py b/bin/doins.py index 98dc4f810..a08e3f8c9 100644 --- a/bin/doins.py +++ b/bin/doins.py @@ -110,10 +110,6 @@ def _parse_install_options( parser.add_argument('-p', '--preserve-timestamps', action='store_true') split_options = shlex.split(options) namespace, remaining = parser.parse_known_args(split_options) - if namespace.preserve_timestamps and sys.version_info < (3, 3): - # -p is not supported in this case, since timestamps cannot - # be preserved with full precision - remaining.append('-p') # Because parsing '--mode' option is partially supported. If unknown # arg for --mode is passed, namespace.mode is set to None. if remaining or namespace.mode is None: @@ -151,15 +147,7 @@ def _set_timestamps(source_stat, dest): source_stat: stat result for the source file. dest: path to the dest file. """ - os.utime(dest, (source_stat.st_atime, source_stat.st_mtime)) - - -if sys.version_info >= (3, 3): - def _set_timestamps_ns(source_stat, dest): - os.utime(dest, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns)) - - _set_timestamps_ns.__doc__ = _set_timestamps.__doc__ - _set_timestamps = _set_timestamps_ns + os.utime(dest, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns)) class _InsInProcessInstallRunner(object): diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py index 322de79ba..a1ba58822 100644 --- a/lib/portage/_emirrordist/FetchTask.py +++ b/lib/portage/_emirrordist/FetchTask.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Gentoo Authors +# Copyright 2013-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from __future__ import division @@ -385,14 +385,9 @@ class FetchTask(CompositeTask): # Apply the timestamp from the source file, but # just rely on umask for permissions. try: - if sys.hexversion >= 0x303: - os.utime(copier.dest_path, - ns=(self._current_stat.st_mtime_ns, - self._current_stat.st_mtime_ns)) - else: - os.utime(copier.dest_path, - (self._current_stat[stat.ST_MTIME], - self._current_stat[stat.ST_MTIME])) + os.utime(copier.dest_path, + ns=(self._current_stat.st_mtime_ns, + self._current_stat.st_mtime_ns)) except OSError as e: msg = "%s %s utime failed unexpectedly: %s" % \ (self.distfile, current_mirror.name, e) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index abb0a9308..2e29b25e5 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -435,10 +435,7 @@ class vardbapi(dbapi)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/_emerge/, lib/portage/util/_async/, ...
commit: dd69ce742c62b9515cf7ae37e46bcf7f178777db Author: Zac Medico gentoo org> AuthorDate: Sun Mar 1 02:17:52 2020 + Commit: Zac Medico gentoo org> CommitDate: Tue Jun 23 02:13:06 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dd69ce74 Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746) This variable specifies a command that filters build log output to a log file. The plan is to extend this to support a separate filter for tty output in the future. In order to enable the EbuildPhase class to write elog messages to the build log with PORTAGE_LOG_FILTER_FILE_CMD support, convert its _elog method to a coroutine, and add a SchedulerInterface async_output method for it to use. Use a new BuildLogger class to manage log output (with or without a filter command), with compression support provided by PipeLogger. BuildLogger has a stdin property which provides access to a writable binary file stream (refers to a pipe) that log content is written to. Bug: https://bugs.gentoo.org/709746 Reviewed-by: Brian Dolbec gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/_emerge/AbstractEbuildProcess.py | 3 +- lib/_emerge/BinpkgFetcher.py | 3 +- lib/_emerge/EbuildFetcher.py | 3 +- lib/_emerge/EbuildPhase.py | 47 +++-- lib/_emerge/SpawnProcess.py| 58 --- lib/portage/dbapi/_MergeProcess.py | 3 +- .../package/ebuild/_config/special_env_vars.py | 8 +- lib/portage/util/_async/BuildLogger.py | 109 + lib/portage/util/_async/SchedulerInterface.py | 32 +- man/make.conf.5| 7 +- 10 files changed, 243 insertions(+), 30 deletions(-) diff --git a/lib/_emerge/AbstractEbuildProcess.py b/lib/_emerge/AbstractEbuildProcess.py index 1c1955cfe..ae1aae55f 100644 --- a/lib/_emerge/AbstractEbuildProcess.py +++ b/lib/_emerge/AbstractEbuildProcess.py @@ -1,4 +1,4 @@ -# Copyright 1999-2019 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -196,6 +196,7 @@ class AbstractEbuildProcess(SpawnProcess): null_fd = os.open('/dev/null', os.O_RDONLY) self.fd_pipes[0] = null_fd + self.log_filter_file = self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD') try: SpawnProcess._start(self) finally: diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index 36d027de3..2e5861cc1 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -1,4 +1,4 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import functools @@ -158,6 +158,7 @@ class _BinpkgFetcherProcess(SpawnProcess): self.env = fetch_env if settings.selinux_enabled(): self._selinux_type = settings["PORTAGE_FETCH_T"] + self.log_filter_file = settings.get('PORTAGE_LOG_FILTER_FILE_CMD') SpawnProcess._start(self) def _pipe(self, fd_pipes): diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py index 1e40994fb..55349c33c 100644 --- a/lib/_emerge/EbuildFetcher.py +++ b/lib/_emerge/EbuildFetcher.py @@ -1,4 +1,4 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import copy @@ -225,6 +225,7 @@ class _EbuildFetcherProcess(ForkProcess): settings["NOCOLOR"] = nocolor self._settings = settings + self.log_filter_file = settings.get('PORTAGE_LOG_FILTER_FILE_CMD') ForkProcess._start(self) # Free settings now since it's no longer needed in diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py index 477e0ba97..ddb3dc719 100644 --- a/lib/_emerge/EbuildPhase.py +++ b/lib/_emerge/EbuildPhase.py @@ -26,6 +26,8 @@ from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir, from portage.util.futures.compat_coroutine import coroutine from portage.util import writemsg from portage.util._async.AsyncTaskFuture import AsyncTaskFuture +from portage.util._async.BuildLogger import BuildLogger +from portage.util.futures import asyncio from portage.util.futures.executor.fork import ForkExecutor try: @@ -69,6 +71,11 @@ class EbuildPhase(CompositeTask): _locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm") def _start(self): + future = asyncio.ensure_future(self._async_start(), loop=self.scheduler) + self._start_task(AsyncTaskFuture(future=future), self._async_start_exit) + + @coroutine + de
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 976c1133100f6da81cd8d6e13f8a723a9fc7cd85 Author: Zac Medico gentoo org> AuthorDate: Sun Jun 7 02:25:11 2020 + Commit: Zac Medico gentoo org> CommitDate: Sun Jun 7 02:53:27 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=976c1133 _better_cache._scan_cat: avoid stat calls (bug 725934) When processing category listdir results, do not use os.path.isdir to identify packages, in order to avoid unecessary stat calls. Instead, use the Atom class to validate package names. This may cause creation of some cache entries for non-packages, but it will not do any harm since these entries will never be accessed via the __getitem__ method. Bug: https://bugs.gentoo.org/725934 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/porttree.py | 9 +++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 08af17bcd..ed992d1e2 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -164,8 +164,13 @@ class _better_cache(object): raise continue for p in pkg_list: - if os.path.isdir(cat_dir + "/" + p): - self._items[cat + "/" + p].append(repo) + try: + atom = Atom("%s/%s" % (cat, p)) + except InvalidAtom: + continue + if atom != atom.cp: + continue + self._items[atom.cp].append(repo) self._scanned_cats.add(cat)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 3de00842d69669a15adaf9e81b0007a7052da5d0 Author: Zac Medico gentoo org> AuthorDate: Thu Feb 20 09:51:34 2020 + Commit: Zac Medico gentoo org> CommitDate: Thu Feb 20 09:52:33 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3de00842 async_fetch_map: handle _parse_uri_map exception (bug 710130) If _parse_uri_map raises an exception then raise it to the caller via the returned future. Bug: https://bugs.gentoo.org/710130 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/porttree.py | 9 ++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 4bb396a6c..08af17bcd 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2019 Gentoo Authors +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -804,8 +804,11 @@ class portdbapi(dbapi): (mypkg, eapi))) return - result.set_result(_parse_uri_map(mypkg, - {'EAPI':eapi,'SRC_URI':myuris}, use=useflags)) + try: + result.set_result(_parse_uri_map(mypkg, + {'EAPI':eapi,'SRC_URI':myuris}, use=useflags)) + except Exception as e: + result.set_exception(e) aux_get_future = self.async_aux_get( mypkg, ["EAPI", "SRC_URI"], mytree=mytree, loop=loop)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 6d8104e665b55174980cd88695f0f956935507b7 Author: Zac Medico gentoo org> AuthorDate: Mon Feb 3 03:01:06 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Feb 3 03:02:02 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6d8104e6 sphinx-build: remove broken lazyimport for _spawn_phase Fixes: d318bcce356b ("vartree: add missing _merge_unicode_error import") Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 29843656f..050366528 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -23,7 +23,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.locks:lockdir,unlockdir,lockfile,unlockfile', 'portage.output:bold,colorize', 'portage.package.ebuild.doebuild:doebuild_environment,' + \ - '_merge_unicode_error', '_spawn_phase', + '_merge_unicode_error', 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', 'portage.process:find_binary',
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: f90400eedc6a8788878c050880db564800c825d2 Author: Zac Medico gentoo org> AuthorDate: Fri Aug 23 20:00:15 2019 + Commit: Zac Medico gentoo org> CommitDate: Sat Aug 24 02:41:35 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f90400ee preserve-libs: ignore dropped non-soname symlink (bug 692698) Fix the dblink _find_libs_to_preserve method to ignore a dropped non-soname symlink. For example, pam-1.3.1-r1 drops the non-soname symlink named libpam_misc.so, and we don't want this to trigger unnecessary preservation of the corresponding library, since the corresponding libpam_misc.so.0 soname symlink and the hardlink that it references are still provided by pam-1.3.1-r1. Bug: https://bugs.gentoo.org/692698 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 15 +++ 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 4f069474b..fa1e1523c 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -3133,10 +3133,6 @@ class dblink(object): os = portage.os f = f_abs[root_len:] - if not unmerge and self.isowner(f): - # We have an indentically named replacement file, - # so we don't try to preserve the old copy. - continue try: consumers = linkmap.findConsumers(f, exclude_providers=(installed_instance.isowner,)) @@ -3184,16 +3180,27 @@ class dblink(object): hardlinks = set() soname_symlinks = set() soname = linkmap.getSoname(next(iter(preserve_node.alt_paths))) + have_replacement_soname_link = False + have_replacement_hardlink = False for f in preserve_node.alt_paths: f_abs = os.path.join(root, f.lstrip(os.sep)) try: if stat.S_ISREG(os.lstat(f_abs).st_mode): hardlinks.add(f) + if not unmerge and self.isowner(f): + have_replacement_hardlink = True + if os.path.basename(f) == soname: + have_replacement_soname_link = True elif os.path.basename(f) == soname: soname_symlinks.add(f) + if not unmerge and self.isowner(f): + have_replacement_soname_link = True except OSError: pass + if have_replacement_hardlink and have_replacement_soname_link: + continue + if hardlinks: preserve_paths.update(hardlinks) preserve_paths.update(soname_symlinks)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/
commit: 0b7eda500a0dcb98a67f33bf9ef25b202b358986 Author: Arfrever Frehtes Taifersar Arahesis Apache Org> AuthorDate: Wed Aug 7 17:06:11 2019 + Commit: Zac Medico gentoo org> CommitDate: Sun Aug 11 19:11:47 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0b7eda50 dblink._collision_protect: Detect internal collisions. Implement detection of internal collisions (between files of the same package, located in separate directories in the installation image (${D}) corresponding to merged directories in the target filesystem (${ROOT})). This provides protection against overwriting some files when performing merging of files from ${D} to ${ROOT} in some filesystem layouts (such as /-merged layout or /usr-merged layout). Internal collisions between identical files are silently ignored. Bug: https://bugs.gentoo.org/690484 Signed-off-by: Arfrever Frehtes Taifersar Arahesis Apache.Org> Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/vartree.py | 55 ++-- lib/portage/util/_compare_files.py | 103 + 2 files changed, 154 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index e2fce7736..4f069474b 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -30,6 +30,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ 'grabdict,normalize_path,new_protect_filename', + 'portage.util._compare_files:compare_files', 'portage.util.digraph:digraph', 'portage.util.env_update:env_update', 'portage.util.install_mask:install_mask_dir,InstallMask', @@ -87,6 +88,7 @@ import io from itertools import chain import logging import os as _os +import operator import platform import pwd import re @@ -3418,6 +3420,8 @@ class dblink(object): os = _os_merge + real_relative_paths = {} + collision_ignore = [] for x in portage.util.shlex_split( self.settings.get("COLLISION_IGNORE", "")): @@ -3469,8 +3473,13 @@ class dblink(object): previous = current progress_shown = True - dest_path = normalize_path( - os.path.join(destroot, f.lstrip(os.path.sep))) + dest_path = normalize_path(os.path.join(destroot, f.lstrip(os.path.sep))) + + # Relative path with symbolic links resolved only in parent directories + real_relative_path = os.path.join(os.path.realpath(os.path.dirname(dest_path)), + os.path.basename(dest_path))[len(destroot):] + + real_relative_paths.setdefault(real_relative_path, []).append(f.lstrip(os.path.sep)) parent = os.path.dirname(dest_path) if parent not in dirs: @@ -3556,9 +3565,24 @@ class dblink(object): break if stopmerge: collisions.append(f) + + internal_collisions = {} + for real_relative_path, files in real_relative_paths.items(): + # Detect internal collisions between non-identical files. + if len(files) >= 2: + files.sort() + for i in range(len(files) - 1): + file1 = normalize_path(os.path.join(srcroot, files[i])) + file2 = normalize_path(os.path.join(srcroot, files[i+1])) + # Compare files, ignoring differences in times. + differences = compare_files(file1, file2, skipped_types=("atime", "mtime", "ctime")) + if differences: + internal_collisions.setdefault(real_relative_path, {})[(files[i], files[i+1])] = differences + if progress_shown: showMessage(_("100% done\n")) - return collisions, dirs_ro, symlink_collisions, plib_collisions + + return collisions, internal_collisions, dirs_ro, symlink_collisions, plib_collisions def _lstat_inode_map(self, path_iter): """ @@ -4081,7 +4105,7 @@ class d
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 611a53a6943bf25ac8c1d909e34e4b37c8873e83 Author: Zac Medico gentoo org> AuthorDate: Thu Jun 20 01:37:53 2019 + Commit: Zac Medico gentoo org> CommitDate: Thu Jun 20 02:21:36 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=611a53a6 portdbapi.getfetchsizes: use .__download__ suffix If the file does not exist locally or the size is not correct, use the file with the .__download__ suffix to compute the remaining_size to fetch. Fixes: ebbde237d33e ("fetch: atomic downloads (bug 175612)") Bug: https://bugs.gentoo.org/688124 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/porttree.py | 12 1 file changed, 12 insertions(+) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index edff0c2f2..e4b688b9a 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -14,6 +14,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.dbapi.dep_expand:dep_expand', 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot', 'portage.package.ebuild.doebuild:doebuild', + 'portage.package.ebuild.fetch:_download_suffix', 'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level', 'portage.util.listdir:listdir', 'portage.versions:best,catsplit,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str', @@ -844,6 +845,17 @@ class portdbapi(dbapi): mystat = os.stat(file_path) except OSError: pass + else: + if mystat.st_size != fetch_size: + # Use file with _download_suffix instead. + mystat = None + + if mystat is None: + try: + mystat = os.stat(file_path + _download_suffix) + except OSError: + pass + if mystat is None: existing_size = 0 ro_distdirs = self.settings.get("PORTAGE_RO_DISTDIRS")
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/futures/_asyncio/
commit: 00340dfa7d85144aa975079d6eeebaa035178208 Author: Zac Medico gentoo org> AuthorDate: Sun May 12 06:51:49 2019 + Commit: Zac Medico gentoo org> CommitDate: Sun May 12 07:02:03 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=00340dfa portdbapi._event_loop: split out _safe_loop function Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/porttree.py | 10 +- lib/portage/util/futures/_asyncio/__init__.py | 18 ++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 7a28f5876..edff0c2f2 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -34,7 +34,6 @@ from portage import eclass_cache, \ from portage import os from portage import _encodings from portage import _unicode_encode -from portage.util._eventloop.EventLoop import EventLoop from portage.util.futures import asyncio from portage.util.futures.compat_coroutine import coroutine, coroutine_return from portage.util.futures.iter_completed import iter_gather @@ -346,14 +345,7 @@ class portdbapi(dbapi): @property def _event_loop(self): - if portage._internal_caller: - # For internal portage usage, asyncio._wrap_loop() is safe. - return asyncio._wrap_loop() - else: - # For external API consumers, use a local EventLoop, since - # we don't want to assume that it's safe to override the - # global SIGCHLD handler. - return EventLoop(main=False) + return asyncio._safe_loop() def _create_pregen_cache(self, tree): conf = self.repositories.get_repo_for_location(tree) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index 2a637624d..e77c7a690 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -37,6 +37,7 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.futures.unix_events:_PortageEventLoopPolicy', 'portage.util.futures:compat_coroutine@_compat_coroutine', + 'portage.util._eventloop.EventLoop:EventLoop@_EventLoop', ) from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as _AsyncioEventLoop from portage.util._eventloop.global_event_loop import ( @@ -250,3 +251,20 @@ if _asyncio_enabled: loop = loop or _global_event_loop() return (loop if hasattr(loop, '_asyncio_wrapper') else _AsyncioEventLoop(loop=loop)) + + +def _safe_loop(): + """ + Return an event loop that's safe to use within the current context. + For portage internal callers, this returns a globally shared event + loop instance. For external API consumers, this constructs a + temporary event loop instance that's safe to use in a non-main + thread (it does not override the global SIGCHLD handler). + + @rtype: asyncio.AbstractEventLoop (or compatible) + @return: event loop instance + """ + if portage._internal_caller: + return _wrap_loop() + else: + return _EventLoop(main=False)
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: fbebef9677d63db70f1c68b197e58b041ec6ac61 Author: Zac Medico gentoo org> AuthorDate: Thu May 9 20:10:46 2019 + Commit: Zac Medico gentoo org> CommitDate: Sat May 11 20:55:49 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fbebef96 repoman: check IUSE in _match_use for USE defaults (bug 685482) This corrects a problem triggered with USE defaults where repoman returns an incorrect negative match for an atom with USE defaults. For example, it triggered this dependency.bad error: RepoMan scours the neighborhood... dependency.bad [fatal]2 dev-libs/libxml2/libxml2-2.9.9-r1.ebuild: DEPEND: ~riscv(default/linux/riscv/17.0/rv64gc) [ '>=dev-lang/python-exec-2:=[python_targets_python2_7(-)?,python_targets_python3_5(-)?,python_targets_python3_6(-)?,python_targets_python3_7(-)?,-python_single_target_python2_7(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-),-python_single_target_python3_7(-)]'] dev-libs/libxml2/libxml2-2.9.9-r1.ebuild: RDEPEND: ~riscv(default/linux/riscv/17.0/rv64gc) [ '>=dev-lang/python-exec-2:=[python_targets_python2_7(-)?,python_targets_python3_5(-)?,python_targets_python3_6(-)?,python_targets_python3_7(-)?,-python_single_target_python2_7(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-),-python_single_target_python3_7(-)]'] State in the _match_use method looked like this: (Pdb) atom '>=dev-lang/python-exec-2:=[python_targets_python3_7(-),-python_single_target_python2_7(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-),-python_single_target_python3_7(-)]' (Pdb) useforce frozenset({'python_targets_pypy', 'python_targets_pypy3', 'python_targets_python3_5', 'big-endian', 'userland_GNU', 'python_targets_python3_7', 'riscv', 'python_single_target_python3_7', 'python_targets_python2_7', 'elibc_glibc', 'python_targets_python3_6', 'abi_riscv_lp64d', 'kernel_linux', 'python_targets_jython2_7'}) (Pdb) atom.use.disabled frozenset({'python_single_target_python3_6', 'python_single_target_python2_7', 'python_single_target_python3_7', 'python_single_target_python3_5'}) (Pdb) atom.use.missing_disabled frozenset({'python_single_target_python2_7', 'python_targets_python3_5', 'python_targets_python3_7', 'python_single_target_python3_5', 'python_single_target_python3_7', 'python_targets_python2_7', 'python_targets_python3_6', 'python_single_target_python3_6'}) Bug: https://bugs.gentoo.org/685482 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py index 6fca6090c..80f8a689f 100644 --- a/lib/portage/dbapi/__init__.py +++ b/lib/portage/dbapi/__init__.py @@ -314,12 +314,12 @@ class dbapi(object): # Check masked and forced flags for repoman. usemask = self.settings._getUseMask(pkg, stable=self.settings._parent_stable) - if any(x in usemask for x in atom.use.enabled): + if any(x in usemask and iuse.get_real_flag(x) is not None for x in atom.use.enabled): return False useforce = self.settings._getUseForce(pkg, stable=self.settings._parent_stable) - if any(x in useforce and x not in usemask + if any(x in useforce and x not in usemask and iuse.get_real_flag(x) is not None for x in atom.use.disabled): return False
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/futures/executor/, ...
commit: 3888e16314afa9107aa0f2d9dce864b85246822c Author: Robin H. Johnson gentoo org> AuthorDate: Sat Feb 16 06:17:12 2019 + Commit: Robin H. Johnson gentoo org> CommitDate: Sat Feb 16 06:17:12 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3888e163 Replace multiprocessing.cpu_count with portage.util.cpuinfo.get_cpu_count portage.util.cpuinfo.get_cpu_count was only used in one spot before, and other call-sites just used multiprocessing.cpu_count() directly. Replace all multiprocessing.cpu_count() calls with get_cpu_count() in portage.util.cpuinfo, to ensure consistency in CPU calculation. Signed-off-by: Robin H. Johnson gentoo.org> lib/portage/dbapi/porttree.py | 4 ++-- lib/portage/util/futures/executor/fork.py | 4 ++-- lib/portage/util/futures/iter_completed.py | 18 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 2ff3e1b34..64a5f3681 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -1471,11 +1471,11 @@ def _async_manifest_fetchlist(portdb, repo_config, cp, cpv_list=None, @param cpv_list: list of ebuild cpv values for a Manifest @type cpv_list: list @param max_jobs: max number of futures to process concurrently (default - is multiprocessing.cpu_count()) + is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default - is multiprocessing.cpu_count()) + is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop diff --git a/lib/portage/util/futures/executor/fork.py b/lib/portage/util/futures/executor/fork.py index 72844403c..add7b3c9e 100644 --- a/lib/portage/util/futures/executor/fork.py +++ b/lib/portage/util/futures/executor/fork.py @@ -7,13 +7,13 @@ __all__ = ( import collections import functools -import multiprocessing import os import sys import traceback from portage.util._async.AsyncFunction import AsyncFunction from portage.util.futures import asyncio +from portage.util.cpuinfo import get_cpu_count class ForkExecutor(object): @@ -24,7 +24,7 @@ class ForkExecutor(object): This is entirely driven by an event loop. """ def __init__(self, max_workers=None, loop=None): - self._max_workers = max_workers or multiprocessing.cpu_count() + self._max_workers = max_workers or get_cpu_count() self._loop = asyncio._wrap_loop(loop) self._submit_queue = collections.deque() self._running_tasks = {} diff --git a/lib/portage/util/futures/iter_completed.py b/lib/portage/util/futures/iter_completed.py index 31b5e0c78..4c48ea0fe 100644 --- a/lib/portage/util/futures/iter_completed.py +++ b/lib/portage/util/futures/iter_completed.py @@ -2,11 +2,11 @@ # Distributed under the terms of the GNU General Public License v2 import functools -import multiprocessing from portage.util._async.AsyncTaskFuture import AsyncTaskFuture from portage.util._async.TaskScheduler import TaskScheduler from portage.util.futures import asyncio +from portage.util.cpuinfo import get_cpu_count def iter_completed(futures, max_jobs=None, max_load=None, loop=None): @@ -18,11 +18,11 @@ def iter_completed(futures, max_jobs=None, max_load=None, loop=None): @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default - is multiprocessing.cpu_count()) + is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default - is multiprocessing.cpu_count()) + is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @param loop: event loop @type loop: EventLoop @@ -47,11 +47,11 @@ def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None): @param futures: iterator of asyncio.Future (or compatible) @type futures: iterator @param max_jobs: max number of futures to process concurrently (default - is multiprocessing.cpu_count()) + is portage.util.cpuinfo.get_cpu_count()) @type max_jobs: int @param max_load: max load allowed when scheduling a new future, otherwise schedule no more than 1 future at a time (default - is multiprocessing.cpu_count()) + is portage.util.cpuinfo.get_cpu_count()) @type max_load: int or float @pa
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: bbe00f515b80be3d6e279e4033f1cbc776a7fe27 Author: Zac Medico gentoo org> AuthorDate: Sun Jan 20 06:39:39 2019 + Commit: Zac Medico gentoo org> CommitDate: Sun Jan 20 06:53:34 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bbe00f51 portagetree: getname and portroot DeprecationWarning Bug: https://bugs.gentoo.org/668164 Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/porttree.py | 16 ++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 76b7967f7..2ff3e1b34 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -1339,10 +1339,18 @@ class portagetree(object): " constructor is unused", DeprecationWarning, stacklevel=2) - self.portroot = settings["PORTDIR"] self.__virtual = virtual self.dbapi = portdbapi(mysettings=settings) + @property + def portroot(self): + """Deprecated. Use the portdbapi getRepositoryPath method instead.""" + warnings.warn("The portroot attribute of " + "portage.dbapi.porttree.portagetree is deprecated. Use the " + "portdbapi getRepositoryPath method instead.", + DeprecationWarning, stacklevel=3) + return self.settings['PORTDIR'] + @property def root(self): warnings.warn("The root attribute of " + \ @@ -1383,7 +1391,11 @@ class portagetree(object): return self.dbapi.cp_all() def getname(self, pkgname): - "returns file location for this particular package (DEPRECATED)" + """Deprecated. Use the portdbapi findname method instead.""" + warnings.warn("The getname method of " + "portage.dbapi.porttree.portagetree is deprecated. " + "Use the portdbapi findname method instead.", + DeprecationWarning, stacklevel=2) if not pkgname: return "" mysplit = pkgname.split("/")
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: cfa915d0d575379df4b9f17fd2db3594155861ca Author: Fabian Groffen gentoo org> AuthorDate: Mon Jan 7 14:19:15 2019 + Commit: Fabian Groffen gentoo org> CommitDate: Fri Jan 11 10:14:22 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cfa915d0 collision_protect: use dynamic report interval The reporting of files remaining can look somewhat odd since the report interval is hardcoded to be per 1000 objects. Adjust this interval to be time based. This means that modern (fast) machines likely will never see the countdown messages at all. On slow setups the message will be informative that there is progress, albeit rather slowly. While at it, report percentage done. Output before this patch: * checking 6158 files for package collisions 5158 files remaining ... 4158 files remaining ... 3158 files remaining ... 2158 files remaining ... 1158 files remaining ... 158 files remaining ... Possible output after this patch on a slower machine: * checking 6158 files for package collisions 48% done, 3145 files remaining ... 96% done, 192 files remaining ... 100% done Signed-off-by: Fabian Groffen gentoo.org> lib/portage/dbapi/vartree.py | 15 +-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 9febf0c71..63389f9a3 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -35,6 +35,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.install_mask:install_mask_dir,InstallMask', 'portage.util.listdir:dircache,listdir', 'portage.util.movefile:movefile', + 'portage.util.monotonic:monotonic', 'portage.util.path:first_existing,iter_parents', 'portage.util.writeable_check:get_ro_checker', 'portage.util._xattr:xattr', @@ -3453,13 +3454,21 @@ class dblink(object): symlink_collisions = [] destroot = self.settings['ROOT'] totfiles = len(file_list) + len(symlink_list) + previous = monotonic() + progress_shown = False + report_interval = 1.7 # seconds + falign = len("%d" % totfiles) showMessage(_(" %s checking %d files for package collisions\n") % \ (colorize("GOOD", "*"), totfiles)) for i, (f, f_type) in enumerate(chain( ((f, "reg") for f in file_list), ((f, "sym") for f in symlink_list))): - if i % 1000 == 0 and i != 0: - showMessage(_("%d files remaining ...\n") % (totfiles - i)) + current = monotonic() + if current - previous > report_interval: + showMessage(_("%3d%% done, %*d files remaining ...\n") % + (i * 100 / totfiles, falign, totfiles - i)) + previous = current + progress_shown = True dest_path = normalize_path( os.path.join(destroot, f.lstrip(os.path.sep))) @@ -3548,6 +3557,8 @@ class dblink(object): break if stopmerge: collisions.append(f) + if progress_shown: + showMessage(_("100% done\n")) return collisions, dirs_ro, symlink_collisions, plib_collisions def _lstat_inode_map(self, path_iter):
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: ebd2c386f5ae2aced4c3ea05dacffdb99bd0bf5b Author: Zac Medico gentoo org> AuthorDate: Mon Sep 24 03:04:13 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Sep 24 06:45:32 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ebd2c386 portdbapi: add async_xmatch method (bug 666940) Add an async_xmatch method, and use it to implement the synchronous xmatch method. Deprecate unused xmatch method parameters. Use the compat_coroutine decorator for backward compatibility with python2.7 (via PEP 342 enhanced generators). Bug: https://bugs.gentoo.org/666940 Suggested-by: Daniel Robbins funtoo.org> Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/porttree.py | 60 +++ 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 56955ec34..76b7967f7 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -37,6 +37,7 @@ from portage import _unicode_encode from portage import OrderedDict from portage.util._eventloop.EventLoop import EventLoop from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine, coroutine_return from portage.util.futures.iter_completed import iter_gather from _emerge.EbuildMetadataPhase import EbuildMetadataPhase @@ -1055,8 +1056,20 @@ class portdbapi(dbapi): self._better_cache = None self.frozen = 0 - def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None): - "caching match function; very trick stuff" + def xmatch(self, level, origdep, mydep=DeprecationWarning, + mykey=DeprecationWarning, mylist=DeprecationWarning): + """ + Caching match function. + + @param level: xmatch level (bestmatch-visible, match-all-cpv-only + match-allmatch-visible, minimum-all, minimum-all-ignore-profile, + or minimum-visible) + @type level: str + @param origdep: dependency to match (may omit category) + @type origdep: portage.dep.Atom or str + @return: match result(s) + @rtype: _pkg_str or list of _pkg_str (depends on level) + """ if level == "list-visible": level = "match-visible" warnings.warn("The 'list-visible' mode of " @@ -1064,21 +1077,46 @@ class portdbapi(dbapi): "has been renamed to match-visible", DeprecationWarning, stacklevel=2) - if mydep is None: - #this stuff only runs on first call of xmatch() - #create mydep, mykey from origdep - mydep = dep_expand(origdep, mydb=self, settings=self.settings) - mykey = mydep.cp + if mydep is not DeprecationWarning: + warnings.warn("The 'mydep' parameter of " + "portage.dbapi.porttree.portdbapi.xmatch" + " is deprecated and ignored", + DeprecationWarning, stacklevel=2) + + loop = self._event_loop + return loop.run_until_complete( + self.async_xmatch(level, origdep, loop=loop)) + + @coroutine + def async_xmatch(self, level, origdep, loop=None): + """ + Asynchronous form of xmatch. + + @param level: xmatch level (bestmatch-visible, match-all-cpv-only + match-allmatch-visible, minimum-all, minimum-all-ignore-profile, + or minimum-visible) + @type level: str + @param origdep: dependency to match (may omit category) + @type origdep: portage.dep.Atom or str + @param loop: event loop (defaults to global event loop) + @type loop: EventLoop + @return: match result(s) + @rtype: asyncio.Future (or compatible), which results in a _pkg_str + or list of _pkg_str (depends on level) + """ + mydep = dep_expand(origdep, mydb=self, settings=self.settings) + mykey = mydep.cp #if no updates are being made to the tree, we can consult our xcache... cache_key = None if self.frozen: cache_key = (mydep, mydep.unevaluated_atom) try: - return self.xcache[level][cache_key][:] + coroutine_return(self.xcache[level][cache_key][:]) except KeyError: pass + loop = asyncio._wrap_loop(loop) myval = Non
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: d9151dfb8144563119f016b439de2b0f53d8858e Author: Zac Medico gentoo org> AuthorDate: Mon Sep 24 00:20:47 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Sep 24 00:45:40 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d9151dfb portdbapi.xmatch: remove deprecated *match-list The bestmatch-list and match-list level arguments have been deprecated since v2.2.0, commit 0d375f1105ad67c8b7e3b16b57cdbb367f99cd69. Signed-off-by: Zac Medico gentoo.org> lib/portage/dbapi/porttree.py | 12 1 file changed, 12 deletions(-) diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index aa8b50a57..56955ec34 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -1165,18 +1165,6 @@ class portdbapi(dbapi): else: myval = "" - elif level == "bestmatch-list": - #dep match -- find best match but restrict search to sublist - warnings.warn("The 'bestmatch-list' mode of " - "portage.dbapi.porttree.portdbapi.xmatch is deprecated", - DeprecationWarning, stacklevel=2) - myval = best(list(self._iter_match(mydep, mylist))) - elif level == "match-list": - #dep match -- find all matches but restrict search to sublist (used in 2nd half of visible()) - warnings.warn("The 'match-list' mode of " - "portage.dbapi.porttree.portdbapi.xmatch is deprecated", - DeprecationWarning, stacklevel=2) - myval = list(self._iter_match(mydep, mylist)) else: raise AssertionError( "Invalid level argument: '%s'" % level)