subject:"\[gentoo\-commits\] proj\/portage\:master commit in\: lib\/portage\/dbapi\/, \/"

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2024-03-02 Thread Zac Medico

commit: 62ee9bf8c680b2a18713da5bd453e3a771257409
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Mar  2 22:48:54 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Mar  2 22:50:54 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=62ee9bf8

binarytree._populate_remote: Fix UnboundLocalError for binpkg-request-signature

If an InvalidBinaryPackageFormat exception was raised from
get_binpkg_format for binpkg-request-signature then it
triggered an UnboundLocalError here.

Fixes: 445f10f4214c ("Use binpkg extensions and header to get format")
Bug: https://bugs.gentoo.org/926048
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index f4251b47d6..4ba1407cda 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1624,7 +1624,7 @@ class binarytree:
 binpkg_format = get_binpkg_format(
 d.get("PATH"), remote=True
 )
-except InvalidBinaryPackageFormat:
+except InvalidBinaryPackageFormat as e:
 writemsg(
 colorize(
 "WARN",

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/, /, lib/portage/

2024-02-25 Thread Mike Gilbert

commit: 19e27e0415fd321c39104f7d687bcdc4f4132e24
Author: Mike Gilbert  gentoo  org>
AuthorDate: Sun Feb 25 18:10:15 2024 +
Commit: Mike Gilbert  gentoo  org>
CommitDate: Mon Feb 26 04:10:32 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=19e27e04

Add workaround for loading libc on musl

musl libc has no soname, which causes ctypes.util.find_library to fail.
As a fallback, try to load "libc.so".

Signed-off-by: Mike Gilbert  gentoo.org>

 NEWS|  7 +++
 lib/portage/dbapi/_MergeProcess.py  |  4 ++--
 lib/portage/dbapi/_SyncfsProcess.py | 14 --
 lib/portage/process.py  | 16 +---
 lib/portage/util/_ctypes.py | 15 +++
 lib/portage/util/locale.py  |  7 ++-
 6 files changed, 35 insertions(+), 28 deletions(-)

diff --git a/NEWS b/NEWS
index eb84651b53..258d800373 100644
--- a/NEWS
+++ b/NEWS
@@ -6,8 +6,15 @@ Release notes take the form of the following optional 
categories:
 * Bug fixes
 * Cleanups
 
+portage-3.0.64 (UNRELEASED)
+--
+
+Bug fixes:
+
+
 portage-3.0.63 (2024-02-25)
 --
+* ctypes: Add workaround for loading libc on musl
 
 Bug fixes:
 * emerge: Skip installed packages with emptytree in depgraph selection (bug 
#651018).

diff --git a/lib/portage/dbapi/_MergeProcess.py 
b/lib/portage/dbapi/_MergeProcess.py
index dd5ad71cf8..d9ab2b47aa 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -10,7 +10,7 @@ import fcntl
 import portage
 from portage import os, _unicode_decode
 from portage.package.ebuild._ipc.QueryCommand import QueryCommand
-from portage.util._ctypes import find_library
+from portage.util._ctypes import load_libc
 import portage.elog.messages
 from portage.util._async.ForkProcess import ForkProcess
 from portage.util import no_color
@@ -64,7 +64,7 @@ class MergeProcess(ForkProcess):
 # process, so that it's only done once rather than
 # for each child process.
 if platform.system() == "Linux" and "merge-sync" in settings.features:
-find_library("c")
+load_libc()
 
 # Inherit stdin by default, so that the pdb SIGUSR1
 # handler is usable for the subprocess.

diff --git a/lib/portage/dbapi/_SyncfsProcess.py 
b/lib/portage/dbapi/_SyncfsProcess.py
index ddc2240071..300ae53985 100644
--- a/lib/portage/dbapi/_SyncfsProcess.py
+++ b/lib/portage/dbapi/_SyncfsProcess.py
@@ -4,7 +4,7 @@
 import functools
 
 from portage import os
-from portage.util._ctypes import find_library, LoadLibrary
+from portage.util._ctypes import load_libc
 from portage.util._async.ForkProcess import ForkProcess
 
 
@@ -24,15 +24,9 @@ class SyncfsProcess(ForkProcess):
 
 @staticmethod
 def _get_syncfs():
-filename = find_library("c")
-if filename is not None:
-library = LoadLibrary(filename)
-if library is not None:
-try:
-return library.syncfs
-except AttributeError:
-pass
-
+(libc, _) = load_libc()
+if libc is not None:
+return getattr(libc, "syncfs", None)
 return None
 
 @staticmethod

diff --git a/lib/portage/process.py b/lib/portage/process.py
index cc9ed7bf78..b652e32942 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -37,7 +37,7 @@ portage.proxy.lazyimport.lazyimport(
 from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
 from portage.exception import CommandNotFound
 from portage.proxy.objectproxy import ObjectProxy
-from portage.util._ctypes import find_library, LoadLibrary, ctypes
+from portage.util._ctypes import load_libc, LoadLibrary, ctypes
 
 try:
 from portage.util.netlink import RtNetlink
@@ -960,11 +960,9 @@ def _exec(
 have_unshare = False
 libc = None
 if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
-filename = find_library("c")
-if filename is not None:
-libc = LoadLibrary(filename)
-if libc is not None:
-have_unshare = hasattr(libc, "unshare")
+(libc, _) = load_libc()
+if libc is not None:
+have_unshare = hasattr(libc, "unshare")
 
 if not have_unshare:
 # unshare() may not be supported by libc
@@ -1212,11 +1210,7 @@ class _unshare_validator:
 """
 # This ctypes library lookup caches the result for use in the
 # subprocess when the multiprocessing start method is fork.
-filename = find_library("c")
-if filename is None:
-return errno.ENOTSUP
-
-libc = LoadLibrary(filename)
+(libc, filename) = load_libc()
 if libc is None:
 return errno.ENOTSUP
 

diff --git a/lib/portage/util/_ctypes.py b/lib/portage/util/_ctypes.py
index e6d1e327cb..04e965ba92 100644
--- a/lib/portage/util/_ctypes.py
+++ b/lib/portage/util/_ctypes.py
@@

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2024-02-25 Thread Sam James

commit: 0855821572f32e81b031a250f7491f34a2fd4078
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Feb 24 23:29:29 2024 +
Commit: Sam James  gentoo  org>
CommitDate: Sun Feb 25 08:24:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=08558215

dbapi: Fix TypeError when passing Exception to warnings.warn

If an Exception is passed as a message to warnings.warn then
it triggers this TypeError:

if metadata_updates:
try:
aux_update(cpv, metadata_updates)
except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
>   warnings.warn(e)
E   TypeError: expected string or bytes-like object, got 
'CorruptionKeyError'

Fixes: fb1d0a22f657 ("dbapi: KeyError tolerance during package moves")
Bug: https://bugs.gentoo.org/922935
Signed-off-by: Zac Medico  gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1282
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/__init__.py | 11 +++
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 6f95b93a21..9105227c77 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -1,11 +1,12 @@
-# Copyright 1998-2023 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["dbapi"]
 
 import functools
+import logging
 import re
-import warnings
+import sys
 from typing import Any, Dict, List, Optional, Tuple
 from collections.abc import Sequence
 
@@ -429,7 +430,9 @@ class dbapi:
 try:
 aux_update(cpv, metadata_updates)
 except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
-warnings.warn(e)
+logging.warning(
+f"{e.__class__.__name__}: {e}", exc_info=sys.exc_info()
+)
 if onUpdate:
 onUpdate(maxval, i + 1)
 if onProgress:
@@ -477,6 +480,6 @@ class dbapi:
 try:
 self.aux_update(mycpv, mydata)
 except CorruptionKeyError as e:
-warnings.warn(e)
+logging.warning(f"{e.__class__.__name__}: {e}", 
exc_info=sys.exc_info())
 continue
 return moves

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2024-01-15 Thread Sam James

commit: adc0052b2c022851458f788868e6b194ed1cfe9f
Author: Sam James  gentoo  org>
AuthorDate: Tue Jan 16 07:51:47 2024 +
Commit: Sam James  gentoo  org>
CommitDate: Tue Jan 16 07:52:11 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=adc0052b

bintree: add missing newlines to signed binpkg update notice

Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/bintree.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index ee574f435f..f4251b47d6 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -306,7 +306,7 @@ class bindbapi(fakedbapi):
 colorize(
 "WARN",
 f"Binpkg update ignored for signed package: 
{binpkg_path}, "
-"the file will be removed.",
+"the file will be removed.\n",
 )
 )
 self.bintree.remove(cpv)
@@ -734,7 +734,7 @@ class binarytree:
 writemsg(
 colorize(
 "WARN",
-f"Binpkg update ignored for signed package: 
{binpkg_path}",
+f"Binpkg update ignored for signed package: 
{binpkg_path}\n",
 )
 )
 continue

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2024-01-15 Thread Zac Medico

commit: 68f4ea8a90d8759a1aa859d9188017e21797bdd0
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Jan 16 01:08:36 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Jan 16 05:25:20 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=68f4ea8a

Handle SignatureException during package moves

Ignore package moves for packages that raise SignatureException,
just as they are ignored for packages that have a valid signature.

Bug: https://bugs.gentoo.org/922142
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 16 
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 2342d571b9..ee574f435f 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -296,8 +296,12 @@ class bindbapi(fakedbapi):
 encoding_key = True
 elif binpkg_format == "gpkg":
 mybinpkg = portage.gpkg.gpkg(self.settings, cpv_str, binpkg_path)
-mydata = mybinpkg.get_metadata()
-if mybinpkg.signature_exist:
+try:
+mydata = mybinpkg.get_metadata()
+signature_exist = mybinpkg.signature_exist
+except SignatureException:
+signature_exist = True
+if signature_exist:
 writemsg(
 colorize(
 "WARN",
@@ -721,8 +725,12 @@ class binarytree:
 decode_metadata_name = False
 elif binpkg_format == "gpkg":
 mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, binpkg_path)
-mydata = mybinpkg.get_metadata()
-if mybinpkg.signature_exist:
+try:
+mydata = mybinpkg.get_metadata()
+signature_exist = mybinpkg.signature_exist
+except SignatureException:
+signature_exist = True
+if signature_exist:
 writemsg(
 colorize(
 "WARN",

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2024-01-15 Thread Sam James

commit: c9ed8136f9f0c56dc7b72d400eaa0acbd338778f
Author: Konstantin Tokarev  yandex  ru>
AuthorDate: Fri Jan 12 17:47:45 2024 +
Commit: Sam James  gentoo  org>
CommitDate: Tue Jan 16 05:16:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c9ed8136

vartree: _needs_move() should replace file if filecmp fails for whatever reason

When trying to update I've got "OSError: [Errno 5] Input/output error"
exception traceback from portage originating from _needs_move(). It turned
out that 3 files in my root filesystem were corrupted, which caused filecmp
to fail. This patch makes portage replace original file in this case.

Bug: https://bugs.gentoo.org/722270
Signed-off-by: Konstantin Tokarev  yandex.ru>
Closes: https://github.com/gentoo/portage/pull/1233
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/vartree.py | 14 +-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 5d39ca1965..a00c731c57 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -6283,7 +6283,19 @@ class dblink:
 if not _cmpxattr(src_bytes, dest_bytes, exclude=excluded_xattrs):
 return True
 
-return not filecmp.cmp(src_bytes, dest_bytes, shallow=False)
+try:
+files_equal = filecmp.cmp(src_bytes, dest_bytes, shallow=False)
+except Exception as e:
+writemsg(
+_(
+"Exception '%s' happened when comparing files %s and %s, 
will replace the latter\n"
+)
+% (e, mysrc, mydest),
+noiselevel=-1,
+)
+return True
+
+return not files_equal
 
 
 def merge(

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2024-01-15 Thread Sam James

commit: 58b094bc79e999e44a5b108e2b7273c164aa906e
Author: Alfred Wingate  protonmail  com>
AuthorDate: Fri Jan  5 19:30:38 2024 +
Commit: Sam James  gentoo  org>
CommitDate: Tue Jan 16 05:16:09 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=58b094bc

bintree: use urllib provided attributes for hostname, user and password

In addition to simplifying it also solves an issue with parsing raw
ipv6 addresses.

Bug: https://bugs.gentoo.org/921400
Signed-off-by: Alfred Wingate  protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1230
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/bintree.py | 18 --
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 5fc4716980..2342d571b9 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1344,23 +1344,13 @@ class binarytree:
 for repo in reversed(list(self._binrepos_conf.values())):
 base_url = repo.sync_uri
 parsed_url = urlparse(base_url)
-host = parsed_url.netloc
+host = parsed_url.hostname or ""
 port = parsed_url.port
-user = None
-passwd = None
-user_passwd = ""
+user = parsed_url.username
+passwd = parsed_url.password
+user_passwd = user + "@" if user else ""
 gpkg_only_warned = False
 
-if "@" in host:
-user, host = host.split("@", 1)
-user_passwd = user + "@"
-if ":" in user:
-user, passwd = user.split(":", 1)
-
-if port is not None:
-port_str = f":{port}"
-if host.endswith(port_str):
-host = host[: -len(port_str)]
 pkgindex_file = os.path.join(
 self.settings["EROOT"],
 CACHE_PATH,

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2024-01-02 Thread Zac Medico

commit: 6c915f3ded10c23127c33b771453d54e3f0d4915
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Jan  3 04:26:36 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jan  3 05:56:59 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c915f3d

binarytree: Avoid unnecessary build id incrementation

When allocating a local file name for a remote package,
prefer re-using the existing build id if the file name
is unused locally, in order to avoid unnecessary build
id incrementation triggered when the _max_build_id
method counts remote build ids. Use the same file name
reservation via open with O_EXCL which has been used
during build_id incrementation since fa901a6510c.

Bug: https://bugs.gentoo.org/921208
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 39 ---
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index d352b6fc0e..5fc4716980 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2023 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["bindbapi", "binarytree"]
@@ -2272,16 +2272,6 @@ class binarytree:
 raise InvalidBinaryPackageFormat(binpkg_format)
 
 def _allocate_filename_multi(self, cpv, remote_binpkg_format=None):
-# First, get the max build_id found when _populate was
-# called.
-max_build_id = self._max_build_id(cpv)
-
-# A new package may have been added concurrently since the
-# last _populate call, so use increment build_id until
-# we locate an unused id.
-pf = catsplit(cpv)[1]
-build_id = max_build_id + 1
-
 if remote_binpkg_format is None:
 try:
 binpkg_format = get_binpkg_format(cpv._metadata["PATH"])
@@ -2299,6 +2289,33 @@ class binarytree:
 else:
 raise InvalidBinaryPackageFormat(binpkg_format)
 
+# If the preferred path is available then return
+# that. This prevents unnecessary build_id incrementation
+# triggered when the _max_build_id method counts remote
+# build ids.
+pf = catsplit(cpv)[1]
+if getattr(cpv, "build_id", False):
+preferred_path = f"{os.path.join(self.pkgdir, cpv.cp, 
pf)}-{cpv.build_id}.{binpkg_suffix}"
+if not os.path.exists(preferred_path):
+try:
+# Avoid races
+ensure_dirs(os.path.dirname(preferred_path))
+with open(preferred_path, "x") as f:
+pass
+except FileExistsError:
+pass
+else:
+return (preferred_path, cpv.build_id)
+
+# First, get the max build_id found when _populate was
+# called.
+max_build_id = self._max_build_id(cpv)
+
+# A new package may have been added concurrently since the
+# last _populate call, so use increment build_id until
+# we locate an unused id.
+build_id = max_build_id + 1
+
 while True:
 filename = (
 f"{os.path.join(self.pkgdir, cpv.cp, 
pf)}-{build_id}.{binpkg_suffix}"

[gentoo-commits] proj/portage:master commit in: /, lib/portage/dbapi/

2024-01-01 Thread Sam James

commit: b071a966357a29e8450688a28ca918166b9e4eb0
Author: Sam James  gentoo  org>
AuthorDate: Tue Jan  2 04:00:02 2024 +
Commit: Sam James  gentoo  org>
CommitDate: Tue Jan  2 04:09:56 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b071a966

bintree: don't call trust helper unless bindb is writable

Followup to 6ae45739e208b7a9d59e0b6056be72a5791aae04. My qualm there wrt
writable was whether or not doing something which mutated state (and therefore
possibly the package list) would be confusing but that doesn't make much sense
for a few reasons.

Anyway, change the test to be not just for no-pretend, but also whether the
bindb is writable too, as pretend is already a proxy for whether we may
not have privileges (I can imagine someone possibly having bindb privileges
but not /etc/portage/gnupg, so better to just head this off entirely).

Bug: https://bugs.gentoo.org/915842
Bug: https://bugs.gentoo.org/920180
Signed-off-by: Sam James  gentoo.org>

 NEWS | 3 ++-
 lib/portage/dbapi/bintree.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 038a4dbf8c..2ee334f20c 100644
--- a/NEWS
+++ b/NEWS
@@ -9,7 +9,8 @@ Release notes take the form of the following optional 
categories:
 portage-3.0.61 (UNRELEASED)
 --
 
-TODO
+Bug fixes:
+* bintree: Don't call trust helper unless bindb is writable (bug #915842, bug 
#920180).
 
 portage-3.0.60 (2024-01-02)
 --

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index a139e37659..d352b6fc0e 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1334,7 +1334,7 @@ class binarytree:
 # when binpackages are involved, not only when we refuse unsigned
 # ones. (If the keys have expired we end up refusing signed but
 # technically invalid packages...)
-if not pretend:
+if not pretend and self.dbapi.writable:
 self._run_trust_helper()
 gpkg_only = True
 else:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/_async/, lib/portage/tests/emerge/, ...

2023-12-26 Thread Zac Medico

commit: 1db44d18578a7aee58449cb97e1991cb06c915c3
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Dec 26 06:42:28 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Dec 26 06:42:28 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1db44d18

bintree: support file scheme for binhost src-uri

Add local file scheme support for binhost src-uri in bintree
and BinpkgFetcher. Make BinpkgFetcher use a coroutine to avoid
using callbacks. Add AsyncTaskFuture isAlive method that
BinpkgFetcher can use to check FileCopier state. Add test
command to test emerge -f --getbinpkgonly with local file
scheme in binrepos.conf.

Also, fix the getbinpkgonly_fetchonly test command so that
it does not rename binhost_dir to pkgdir. It seems like it
should not do this, and it caused the getbinpkgonly_file_uri
command to fail.

Bug: https://bugs.gentoo.org/920537
Signed-off-by: Zac Medico  gentoo.org>

 lib/_emerge/BinpkgFetcher.py   | 117 +++--
 lib/portage/dbapi/bintree.py   |  19 +++--
 lib/portage/tests/emerge/conftest.py   |  21 +-
 lib/portage/util/_async/AsyncTaskFuture.py |   8 +-
 4 files changed, 111 insertions(+), 54 deletions(-)

diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 10f9b6e427..01b2bae637 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -1,8 +1,6 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-import functools
-
 from _emerge.AsynchronousLock import AsynchronousLock
 from _emerge.CompositeTask import CompositeTask
 from _emerge.SpawnProcess import SpawnProcess
@@ -14,6 +12,7 @@ from portage import os
 from portage.binpkg import get_binpkg_format
 from portage.exception import FileNotFound
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._async.FileCopier import FileCopier
 from portage.util._pty import _create_pty_or_pipe
 
 
@@ -40,6 +39,22 @@ class BinpkgFetcher(CompositeTask):
 self.pkg_path = self.pkg_allocated_path + ".partial"
 
 def _start(self):
+self._start_task(
+AsyncTaskFuture(future=self._main(), scheduler=self.scheduler),
+self._main_exit,
+)
+
+async def _main(self) -> int:
+"""
+Main coroutine which saves the binary package to self.pkg_path
+and returns the exit status of the fetcher or copier.
+
+@rtype: int
+@return: Exit status of fetcher or copier.
+"""
+pkg = self.pkg
+bintree = pkg.root_config.trees["bintree"]
+
 fetcher = _BinpkgFetcherProcess(
 background=self.background,
 logfile=self.logfile,
@@ -52,47 +67,67 @@ class BinpkgFetcher(CompositeTask):
 if not self.pretend:
 portage.util.ensure_dirs(os.path.dirname(self.pkg_path))
 if "distlocks" in self.pkg.root_config.settings.features:
-self._start_task(
-AsyncTaskFuture(future=fetcher.async_lock()),
-functools.partial(self._start_locked, fetcher),
-)
-return
-
-self._start_task(fetcher, self._fetcher_exit)
-
-def _start_locked(self, fetcher, lock_task):
-self._assert_current(lock_task)
-if lock_task.cancelled:
-self._default_final_exit(lock_task)
-return
-
-lock_task.future.result()
-self._start_task(fetcher, self._fetcher_exit)
-
-def _fetcher_exit(self, fetcher):
-self._assert_current(fetcher)
-if not self.pretend and fetcher.returncode == os.EX_OK:
-fetcher.sync_timestamp()
-if fetcher.locked:
-self._start_task(
-AsyncTaskFuture(future=fetcher.async_unlock()),
-functools.partial(self._fetcher_exit_unlocked, fetcher),
-)
-else:
-self._fetcher_exit_unlocked(fetcher)
+await fetcher.async_lock()
+
+try:
+if bintree._remote_has_index:
+remote_metadata = bintree._remotepkgs[
+bintree.dbapi._instance_key(pkg.cpv)
+]
+rel_uri = remote_metadata.get("PATH")
+if not rel_uri:
+# Assume that the remote index is out of date. No path 
should
+# never happen in new portage versions.
+rel_uri = pkg.cpv + ".tbz2"
+remote_base_uri = remote_metadata["BASE_URI"]
+uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
+else:
+raise FileNotFound("Binary packages index not found")
 
-def _fetcher_exit_unlocked(self, fetcher, unlock_task=None):
-if unlock_task is not None:
-self._assert_current(unlock_task)
-if unlock_task.cancelled:
-

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-12-09 Thread Zac Medico

commit: cc372a52ee2aba153c0d7c2290306b7f623e8b7c
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Dec  9 23:58:07 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Dec 10 01:27:43 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cc372a52

bindbapi: add SIZE to _pkg_str_aux_keys

The _pkg_str_aux_keys are used inside dbapi.update_ents,
and need to contain SIZE in order for the fakedbapi
_instance_key_multi_instance method to operate correctly.

Incorrect operation of _instance_key_multi_instance could
prevent binarytree.inject from removing an old instance
from its internal state. It could also trigger a KeyError
in bindbapi.aux_update as in bug 918597, since it could
cause binarytree.getname to return a non-existent path.
It could also cause binarytree.getname to return an
existing but incorrect path, which might trigger an
InvalidBinaryPackageFormat exception as in bug 906675.

Bug: https://bugs.gentoo.org/906675
Bug: https://bugs.gentoo.org/918597
Bug: https://bugs.gentoo.org/919668
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index a6e1f9773d..9c9ac66334 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -84,9 +84,12 @@ class bindbapi(fakedbapi):
 _known_keys = frozenset(
 list(fakedbapi._known_keys) + ["CHOST", "repository", "USE"]
 )
+# Must include keys used to create _pkg_str attributes used in
+# the fakedbapi _instance_key_multi_instance method.
 _pkg_str_aux_keys = fakedbapi._pkg_str_aux_keys + (
 "BUILD_ID",
 "BUILD_TIME",
+"SIZE",
 "_mtime_",
 )

[gentoo-commits] proj/portage:master commit in: /, lib/portage/dbapi/, lib/portage/tests/dbapi/, lib/_emerge/, ...

2023-11-06 Thread Sam James

commit: 6ae45739e208b7a9d59e0b6056be72a5791aae04
Author: Sam James  gentoo  org>
AuthorDate: Mon Nov  6 17:07:10 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Mon Nov  6 17:25:04 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6ae45739

bintree: don't call trust helper with --pretend

Trust helpers are likely to need privileges and it's a bit too far for pretend
there, I think. People can run it manually if they want it done for them anyway.

We could check writable instead but I'd like to get a fix in for the regression
first.

Bug: https://bugs.gentoo.org/915842
Signed-off-by: Sam James  gentoo.org>

 NEWS |  3 +++
 lib/_emerge/actions.py   |  3 +++
 lib/portage/dbapi/bintree.py | 14 +++---
 lib/portage/tests/dbapi/test_bintree.py  |  6 --
 lib/portage/tests/emerge/test_actions.py |  4 +++-
 5 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/NEWS b/NEWS
index 89d9335275..654b3175f0 100644
--- a/NEWS
+++ b/NEWS
@@ -17,6 +17,9 @@ Bug fixes:
 
 * Avoid crash with blockers in depgraph for binpkg-respect-use notice (bug 
#916336).
 
+* Don't call trust helper (e.g. getuto) with --getbinpkg --pretend as we may
+  lack privileges to do anything (bug #915842).
+
 portage-3.0.54 (2023-10-25)
 --
 

diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 323fde3767..07d477a046 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -174,6 +174,7 @@ def action_build(
 kwargs["add_repos"] = (quickpkg_vardb,)
 
 try:
+kwargs["pretend"] = "--pretend" in emerge_config.opts
 emerge_config.target_config.trees["bintree"].populate(
 getbinpkgs="--getbinpkg" in emerge_config.opts, **kwargs
 )
@@ -3472,6 +3473,8 @@ def run_action(emerge_config):
 emerge_config.running_config.trees["vartree"].dbapi,
 )
 
+kwargs["pretend"] = "--pretend" in emerge_config.opts
+
 try:
 mytrees["bintree"].populate(
 getbinpkgs="--getbinpkg" in emerge_config.opts,

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 0ecfdc25d0..6446fde95a 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -831,6 +831,7 @@ class binarytree:
 add_repos=(),
 force_reindex=False,
 invalid_errors=True,
+pretend=False,
 ):
 """
 Populates the binarytree with package metadata.
@@ -844,6 +845,10 @@ class binarytree:
 @type add_repos: sequence
 """
 
+# TODO: Should we return here if we're --pretend? On the one hand,
+# people might not want --pretend to affect state. On the other hand,
+# it makes --pretend pretty useless with --getbinpkg as your index will
+# be stale.
 if self._populating:
 return
 
@@ -898,7 +903,9 @@ class binarytree:
 noiselevel=-1,
 )
 else:
-self._populate_remote(getbinpkg_refresh=getbinpkg_refresh)
+self._populate_remote(
+getbinpkg_refresh=getbinpkg_refresh, pretend=pretend
+)
 
 finally:
 self._populating = False
@@ -1290,7 +1297,7 @@ class binarytree:
 return
 ret.check_returncode()
 
-def _populate_remote(self, getbinpkg_refresh=True):
+def _populate_remote(self, getbinpkg_refresh=True, pretend=False):
 self._remote_has_index = False
 self._remotepkgs = {}
 
@@ -1299,7 +1306,8 @@ class binarytree:
 # when binpackages are involved, not only when we refuse unsigned
 # ones. (If the keys have expired we end up refusing signed but
 # technically invalid packages...)
-self._run_trust_helper()
+if not pretend:
+self._run_trust_helper()
 gpkg_only = True
 else:
 gpkg_only = False

diff --git a/lib/portage/tests/dbapi/test_bintree.py 
b/lib/portage/tests/dbapi/test_bintree.py
index 5a6ee5b142..0aa411ad97 100644
--- a/lib/portage/tests/dbapi/test_bintree.py
+++ b/lib/portage/tests/dbapi/test_bintree.py
@@ -120,7 +120,9 @@ class BinarytreeTestCase(TestCase):
 settings.__getitem__.return_value = "/some/path"
 bt = binarytree(pkgdir=os.getenv("TMPDIR", "/tmp"), settings=settings)
 bt.populate(getbinpkgs=True, getbinpkg_refresh=refresh)
-ppopulate_remote.assert_called_once_with(getbinpkg_refresh=refresh)
+ppopulate_remote.assert_called_once_with(
+getbinpkg_refresh=refresh, pretend=False
+)
 
 @patch("portage.dbapi.bintree.writemsg")
 @patch("portage.dbapi.bintree.BinRepoConfigLoader")
@@ -161,4 +163,4 @@ class BinarytreeTestCase(TestCase):

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-10-24 Thread Zac Medico

commit: 7a43de7a8ce8dc6836d998cb87a5135ba8f0bc2c
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Oct 24 18:07:53 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Oct 24 18:08:23 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a43de7a

dbapi: Make _iuse_implicit_cnstr picklable for spawn compat

Bug: https://bugs.gentoo.org/916235
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/__init__.py | 15 +++
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 233da8d698..dd697db109 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -1,8 +1,9 @@
-# Copyright 1998-2020 Gentoo Authors
+# Copyright 1998-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["dbapi"]
 
+import functools
 import re
 import warnings
 from typing import Any, Dict, List, Optional, Tuple
@@ -223,6 +224,10 @@ class dbapi:
 
 yield cpv
 
+@staticmethod
+def _iuse_implicit_built(iuse_implicit_match, use, flag):
+return iuse_implicit_match(flag) or flag in use
+
 def _iuse_implicit_cnstr(self, pkg, metadata):
 """
 Construct a callable that checks if a given USE flag should
@@ -257,9 +262,11 @@ class dbapi:
 # This behavior is only used for EAPIs that support IUSE_EFFECTIVE,
 # since built USE settings for earlier EAPIs may contain a large
 # number of irrelevant flags.
-prof_iuse = iuse_implicit_match
-enabled = frozenset(metadata["USE"].split()).__contains__
-iuse_implicit_match = lambda flag: prof_iuse(flag) or enabled(flag)
+iuse_implicit_match = functools.partial(
+self._iuse_implicit_built,
+iuse_implicit_match,
+frozenset(metadata["USE"].split()),
+)
 
 return iuse_implicit_match

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-10-23 Thread Zac Medico

commit: c7768ae4820201913ba28d5fad441c1acc4659c0
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Oct 23 04:14:05 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Oct 23 04:41:10 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c7768ae4

bindbapi: Fix pickling for MergeProcess

Both the _aux_cache_slot_dict_cache and _instance_key attributes are
unpicklable, so fix __getstate__ to exclude them both from the state.
Override __setstate__ to initialize _instance_key appropriately.
These changes prevent the following error:

AttributeError: Can't pickle local object 
'slot_dict_class..LocalSlotDict'

Fixes: b9a85ff987ea ("MergeProcess: Support QueryCommand with spawn start 
method")
Bug: https://bugs.gentoo.org/916106
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 21 +
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 3f1bf9d5d2..0ecfdc25d0 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -139,14 +139,19 @@ class bindbapi(fakedbapi):
 return self._aux_cache_slot_dict_cache
 
 def __getstate__(self):
-# This attribute is not picklable, but it automatically
-# regenerates after unpickling when set to None.
-_aux_cache_slot_dict = self._aux_cache_slot_dict_cache
-self._aux_cache_slot_dict_cache = None
-try:
-return super().__getstate__()
-finally:
-self._aux_cache_slot_dict_cache = _aux_cache_slot_dict
+state = self.__dict__.copy()
+# These attributes are not picklable, so they are automatically
+# regenerated after unpickling.
+state["_aux_cache_slot_dict_cache"] = None
+state["_instance_key"] = None
+return state
+
+def __setstate__(self, state):
+self.__dict__.update(state)
+if self._multi_instance:
+self._instance_key = self._instance_key_multi_instance
+else:
+self._instance_key = self._instance_key_cpv
 
 @property
 def writable(self):

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-10-22 Thread Zac Medico

commit: b9a85ff987ea677e350fb70e2c3355197ee6c39b
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Oct 22 08:08:02 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Oct 22 21:29:49 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9a85ff9

MergeProcess: Support QueryCommand with spawn start method

Since QueryCommand._db does not propagate to the child process
in MergeProcess with the multiprocessing spawn start method,
handle it by passing QueryCommand._db as an explicit parameter.

Since self.mydbapi can be a bindbapi instance, override the
bindbapi __getstate__ method so that it is picklable (omit
the unpicklable _aux_cache_slot_dict attribute and regenerate
it after unpickling).

Bug: https://bugs.gentoo.org/916106
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/_MergeProcess.py | 19 ---
 lib/portage/dbapi/bintree.py   | 20 ++--
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/lib/portage/dbapi/_MergeProcess.py 
b/lib/portage/dbapi/_MergeProcess.py
index cf8e6513d1..dd5ad71cf8 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -9,6 +9,7 @@ import platform
 import fcntl
 import portage
 from portage import os, _unicode_decode
+from portage.package.ebuild._ipc.QueryCommand import QueryCommand
 from portage.util._ctypes import find_library
 import portage.elog.messages
 from portage.util._async.ForkProcess import ForkProcess
@@ -180,6 +181,15 @@ class MergeProcess(ForkProcess):
 self._dblink = mylink
 self._elog_reader_fd = elog_reader_fd
 
+# Since the entire QueryCommand._db is not required, only pass
+# in tree types that QueryCommand specifically requires.
+child_db = {}
+parent_db = portage.db if QueryCommand._db is None else 
QueryCommand._db
+for root in parent_db:
+child_db[root] = {}
+for tree_type in ("vartree", "porttree"):
+child_db[root][tree_type] = parent_db[root][tree_type]
+
 self.target = functools.partial(
 self._target,
 self._counter,
@@ -192,6 +202,7 @@ class MergeProcess(ForkProcess):
 self.settings,
 self.unmerge,
 self.vartree.dbapi,
+child_db,
 )
 
 pids = super()._spawn(args, fd_pipes, **kwargs)
@@ -223,10 +234,12 @@ class MergeProcess(ForkProcess):
 settings,
 unmerge,
 vardb,
+db,
 ):
-"""
-TODO: Make all arguments picklable for the multiprocessing spawn start 
method.
-"""
+if QueryCommand._db is None:
+# Initialize QueryCommand._db for 
AbstractEbuildProcess/EbuildIpcDaemon
+# when not using the multiprocessing fork start method.
+QueryCommand._db = db
 portage.output.havecolor = not no_color(settings)
 # Avoid wastful updates of the vdb cache.
 vardb._flush_cache_enabled = False

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 421863f3e8..3f1bf9d5d2 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2021 Gentoo Authors
+# Copyright 1998-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["bindbapi", "binarytree"]
@@ -129,8 +129,24 @@ class bindbapi(fakedbapi):
 "USE",
 "_mtime_",
 }
-self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys)
 self._aux_cache = {}
+self._aux_cache_slot_dict_cache = None
+
+@property
+def _aux_cache_slot_dict(self):
+if self._aux_cache_slot_dict_cache is None:
+self._aux_cache_slot_dict_cache = 
slot_dict_class(self._aux_cache_keys)
+return self._aux_cache_slot_dict_cache
+
+def __getstate__(self):
+# This attribute is not picklable, but it automatically
+# regenerates after unpickling when set to None.
+_aux_cache_slot_dict = self._aux_cache_slot_dict_cache
+self._aux_cache_slot_dict_cache = None
+try:
+return super().__getstate__()
+finally:
+self._aux_cache_slot_dict_cache = _aux_cache_slot_dict
 
 @property
 def writable(self):

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-10-22 Thread Zac Medico

commit: 0b9fbe389db3c3b9e625e1f5f526b3a921d2e0ce
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Oct 22 19:57:16 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Oct 22 19:58:20 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0b9fbe38

vardbapi.unpack_contents: Use multiprocessing.Pipe for spawn compat

Bug: https://bugs.gentoo.org/916112
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py | 28 +++-
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index a80d3bc0e3..5638935606 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -101,6 +101,7 @@ import grp
 import io
 from itertools import chain
 import logging
+import multiprocessing
 import os as _os
 import operator
 import pickle
@@ -1094,19 +1095,18 @@ class vardbapi(dbapi):
 )
 if binpkg_format == "xpak":
 tar_cmd = ("tar", "-x", "--xattrs", "--xattrs-include=*", "-C", 
dest_dir)
-pr, pw = os.pipe()
+pr, pw = multiprocessing.Pipe(duplex=False)
 proc = await asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)
-os.close(pr)
-with os.fdopen(pw, "wb", 0) as pw_file:
-excluded_config_files = await loop.run_in_executor(
-ForkExecutor(loop=loop),
-functools.partial(
-self._dblink(cpv).quickpkg,
-pw_file,
-include_config=opts.include_config == "y",
-
include_unmodified_config=opts.include_unmodified_config == "y",
-),
-)
+pr.close()
+excluded_config_files = await loop.run_in_executor(
+ForkExecutor(loop=loop),
+functools.partial(
+self._dblink(cpv).quickpkg,
+pw,
+include_config=opts.include_config == "y",
+include_unmodified_config=opts.include_unmodified_config 
== "y",
+),
+)
 await proc.wait()
 if proc.returncode != os.EX_OK:
 raise PortageException(f"command failed: {tar_cmd}")
@@ -2161,7 +2161,9 @@ class dblink:
 # The tarfile module will write pax headers holding the
 # xattrs only if PAX_FORMAT is specified here.
 with tarfile.open(
-fileobj=output_file,
+fileobj=output_file
+if hasattr(output_file, "write")
+else open(output_file.fileno(), mode="wb", closefd=False),
 mode="w|",
 format=tarfile.PAX_FORMAT if xattrs else 
tarfile.DEFAULT_FORMAT,
 ) as tar:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-10-22 Thread Zac Medico

commit: c41d109b121feb0bf6199b4082a35799fdc547b9
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Oct 22 08:28:39 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Oct 22 08:30:50 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c41d109b

vardbapi.unpack_metadata: Make compatible with spawn start method

This solves the following error with the multiprocessing spawn start
method:

AttributeError: Can't pickle local object 
'vardbapi.unpack_metadata..async_copy'

Bug: https://bugs.gentoo.org/916108
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 88fc525771..a80d3bc0e3 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1003,6 +1003,13 @@ class vardbapi(dbapi):
 pass
 self._bump_mtime(cpv)
 
+@staticmethod
+def _async_copy(dbdir, dest_dir):
+for parent, dirs, files in os.walk(dbdir, onerror=_raise_exc):
+for key in files:
+shutil.copy(os.path.join(parent, key), os.path.join(dest_dir, 
key))
+break
+
 async def unpack_metadata(self, pkg, dest_dir, loop=None):
 """
 Unpack package metadata to a directory. This method is a coroutine.
@@ -1018,14 +1025,9 @@ class vardbapi(dbapi):
 else:
 cpv = pkg.mycpv
 dbdir = self.getpath(cpv)
-
-def async_copy():
-for parent, dirs, files in os.walk(dbdir, onerror=_raise_exc):
-for key in files:
-shutil.copy(os.path.join(parent, key), 
os.path.join(dest_dir, key))
-break
-
-await loop.run_in_executor(ForkExecutor(loop=loop), async_copy)
+await loop.run_in_executor(
+ForkExecutor(loop=loop), self._async_copy, dbdir, dest_dir
+)
 
 async def unpack_contents(
 self,

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-10-19 Thread Zac Medico

commit: d92d69a52b2b127a0934656163f7075015ef7c85
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Oct 17 19:05:07 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Oct 17 19:14:54 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d92d69a5

MergeProcess: Use multiprocessing.Pipe to decouple fd_pipes

Use multiprocessing.Pipe to decouple from the fd_pipes implementation
since that currently only works for the multiprocessing "fork" start
method.

Bug: https://bugs.gentoo.org/915903
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/_MergeProcess.py | 25 ++---
 lib/portage/dbapi/vartree.py   |  2 +-
 2 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/lib/portage/dbapi/_MergeProcess.py 
b/lib/portage/dbapi/_MergeProcess.py
index c1085270da..cf8e6513d1 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -96,7 +96,7 @@ class MergeProcess(ForkProcess):
 self._locked_vdb = False
 
 def _elog_output_handler(self):
-output = self._read_buf(self._elog_reader_fd)
+output = self._read_buf(self._elog_reader_fd.fileno())
 if output:
 lines = _unicode_decode(output).split("\n")
 if len(lines) == 1:
@@ -112,8 +112,8 @@ class MergeProcess(ForkProcess):
 reporter(msg, phase=phase, key=key, out=out)
 
 elif output is not None:  # EIO/POLLHUP
-self.scheduler.remove_reader(self._elog_reader_fd)
-os.close(self._elog_reader_fd)
+self.scheduler.remove_reader(self._elog_reader_fd.fileno())
+self._elog_reader_fd.close()
 self._elog_reader_fd = None
 return False
 
@@ -136,16 +136,15 @@ class MergeProcess(ForkProcess):
 post-fork actions.
 """
 
-elog_reader_fd, elog_writer_fd = os.pipe()
+elog_reader_fd, elog_writer_fd = multiprocessing.Pipe(duplex=False)
 
 fcntl.fcntl(
-elog_reader_fd,
+elog_reader_fd.fileno(),
 fcntl.F_SETFL,
-fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
+fcntl.fcntl(elog_reader_fd.fileno(), fcntl.F_GETFL) | 
os.O_NONBLOCK,
 )
 
 mtime_reader, mtime_writer = multiprocessing.Pipe(duplex=False)
-fd_pipes[mtime_writer.fileno()] = mtime_writer.fileno()
 self.scheduler.add_reader(mtime_reader.fileno(), self._mtime_handler)
 self._mtime_reader = mtime_reader
 
@@ -166,8 +165,7 @@ class MergeProcess(ForkProcess):
 pipe=elog_writer_fd,
 mtime_pipe=mtime_writer,
 )
-fd_pipes[elog_writer_fd] = elog_writer_fd
-self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler)
+self.scheduler.add_reader(elog_reader_fd.fileno(), 
self._elog_output_handler)
 
 # If a concurrent emerge process tries to install a package
 # in the same SLOT as this one at the same time, there is an
@@ -185,7 +183,6 @@ class MergeProcess(ForkProcess):
 self.target = functools.partial(
 self._target,
 self._counter,
-self._elog_reader_fd,
 self._dblink,
 self.infloc,
 self.mydbapi,
@@ -198,7 +195,7 @@ class MergeProcess(ForkProcess):
 )
 
 pids = super()._spawn(args, fd_pipes, **kwargs)
-os.close(elog_writer_fd)
+elog_writer_fd.close()
 mtime_writer.close()
 self._buf = ""
 self._elog_keys = set()
@@ -217,7 +214,6 @@ class MergeProcess(ForkProcess):
 @staticmethod
 def _target(
 counter,
-elog_reader_fd,
 mylink,
 infloc,
 mydbapi,
@@ -231,7 +227,6 @@ class MergeProcess(ForkProcess):
 """
 TODO: Make all arguments picklable for the multiprocessing spawn start 
method.
 """
-os.close(elog_reader_fd)
 portage.output.havecolor = not no_color(settings)
 # Avoid wastful updates of the vdb cache.
 vardb._flush_cache_enabled = False
@@ -301,8 +296,8 @@ class MergeProcess(ForkProcess):
 
 self._unlock_vdb()
 if self._elog_reader_fd is not None:
-self.scheduler.remove_reader(self._elog_reader_fd)
-os.close(self._elog_reader_fd)
+self.scheduler.remove_reader(self._elog_reader_fd.fileno())
+self._elog_reader_fd.close()
 self._elog_reader_fd = None
 if self._elog_keys is not None:
 for key in self._elog_keys:

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 835cbb8092..88fc525771 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -4198,7 +4198,7 @@ class dblink:
 if str_buffer:
 str_buffer = _unicode_encode("".join(str_buffer))
 while str_buffer:
-str_buffer =

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-10-15 Thread Zac Medico

commit: 3e38ae92bdd5b057352a2bcb044fb587b15b25f3
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Oct 15 19:21:44 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Oct 15 21:38:08 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3e38ae92

MergeProcess: Eliminate target arguments that reference self

This improves compatibility with the multiprocessing spawn
start method, by eliminating this error:

AttributeError: Can't pickle local object 
'MergeProcess._start..'

Bug: https://bugs.gentoo.org/914876
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/_MergeProcess.py | 40 +++---
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/lib/portage/dbapi/_MergeProcess.py 
b/lib/portage/dbapi/_MergeProcess.py
index 012435dce3..c1085270da 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -74,20 +74,6 @@ class MergeProcess(ForkProcess):
 self.fd_pipes.setdefault(0, portage._get_stdin().fileno())
 
 self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
-self.target = functools.partial(
-self._target,
-lambda: self._counter,
-lambda: self._elog_reader_fd,
-lambda: self._dblink,
-self.infloc,
-self.mydbapi,
-self.myebuild,
-self.pkgloc,
-self.prev_mtimes,
-self.settings,
-self.unmerge,
-self.vartree.dbapi,
-)
 super()._start()
 
 def _lock_vdb(self):
@@ -195,6 +181,22 @@ class MergeProcess(ForkProcess):
 
 self._dblink = mylink
 self._elog_reader_fd = elog_reader_fd
+
+self.target = functools.partial(
+self._target,
+self._counter,
+self._elog_reader_fd,
+self._dblink,
+self.infloc,
+self.mydbapi,
+self.myebuild,
+self.pkgloc,
+self.prev_mtimes,
+self.settings,
+self.unmerge,
+self.vartree.dbapi,
+)
+
 pids = super()._spawn(args, fd_pipes, **kwargs)
 os.close(elog_writer_fd)
 mtime_writer.close()
@@ -214,9 +216,9 @@ class MergeProcess(ForkProcess):
 
 @staticmethod
 def _target(
-get_counter,
-get_elog_reader_fd,
-get_mylink,
+counter,
+elog_reader_fd,
+mylink,
 infloc,
 mydbapi,
 myebuild,
@@ -229,9 +231,7 @@ class MergeProcess(ForkProcess):
 """
 TODO: Make all arguments picklable for the multiprocessing spawn start 
method.
 """
-os.close(get_elog_reader_fd())
-counter = get_counter()
-mylink = get_mylink()
+os.close(elog_reader_fd)
 portage.output.havecolor = not no_color(settings)
 # Avoid wastful updates of the vdb cache.
 vardb._flush_cache_enabled = False

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-10-08 Thread Zac Medico

commit: fc70635aecbf80d3aeeb9f4e56d396040da09279
Author: Siddhanth Rathod  gmail  com>
AuthorDate: Sun Oct  8 09:25:15 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Oct  8 19:47:56 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fc70635a

Fix: move msg array such that it gets assigned only when needed

Signed-off-by: Siddhanth Rathod  gmail.com>
Closes: https://github.com/gentoo/portage/pull/1124
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 7a4166c120..421863f3e8 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1092,10 +1092,10 @@ class binarytree:
 missing_keys.append("PF")
 if not slot:
 missing_keys.append("SLOT")
-msg = []
-if missing_keys:
-missing_keys.sort()
-if invalid_errors:
+if invalid_errors:
+msg = []
+if missing_keys:
+missing_keys.sort()
 msg.append(
 _("Missing metadata key(s): %s.")
 % ", ".join(missing_keys)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/dbapi/

2023-10-07 Thread Sam James

commit: c53d046c9629d5c3a7841aee4e92ae38c0691e69
Author: Siddhanth Rathod  gmail  com>
AuthorDate: Thu Oct  5 13:22:52 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Sun Oct  8 03:29:46 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c53d046c

dbapi: bintree: introduce invalid_errors

Introduce 'invalid_errors' var to enable suppression of invalid binary error,
for use by gentoolkit.

[sam: See gentoolkit side at https://github.com/gentoo/gentoolkit/pull/35].

Bug: https://bugs.gentoo.org/900224
Signed-off-by: Siddhanth Rathod  gmail.com>
Closes: https://github.com/gentoo/portage/pull/1123
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/bintree.py| 60 +++--
 lib/portage/tests/dbapi/test_bintree.py |  6 ++--
 2 files changed, 39 insertions(+), 27 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 580ce2f290..7a4166c120 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -809,6 +809,7 @@ class binarytree:
 getbinpkg_refresh=False,
 add_repos=(),
 force_reindex=False,
+invalid_errors=True,
 ):
 """
 Populates the binarytree with package metadata.
@@ -839,7 +840,8 @@ class binarytree:
 try:
 update_pkgindex = self._populate_local(
 reindex="pkgdir-index-trusted" not in self.settings.features
-or force_reindex
+or force_reindex,
+invalid_errors=invalid_errors,
 )
 
 if update_pkgindex and self.dbapi.writable:
@@ -882,7 +884,7 @@ class binarytree:
 
 self.populated = True
 
-def _populate_local(self, reindex=True):
+def _populate_local(self, reindex=True, invalid_errors=True):
 """
 Populates the binarytree with local package metadata.
 
@@ -1019,11 +1021,15 @@ class binarytree:
 self.dbapi.cpv_inject(mycpv)
 continue
 if not os.access(full_path, os.R_OK):
-writemsg(
-_("!!! Permission denied to read " "binary 
package: '%s'\n")
-% full_path,
-noiselevel=-1,
-)
+if invalid_errors:
+writemsg(
+_(
+"!!! Permission denied to read "
+"binary package: '%s'\n"
+)
+% full_path,
+noiselevel=-1,
+)
 self.invalids.append(myfile[:-5])
 self.invalid_paths[myfile] = [full_path]
 continue
@@ -1062,10 +1068,11 @@ class binarytree:
 binpkg_format=binpkg_format,
 )
 except (PortagePackageException, SignatureException) as e:
-writemsg(
-f"!!! Invalid binary package: '{full_path}', 
{e}\n",
-noiselevel=-1,
-)
+if invalid_errors:
+writemsg(
+f"!!! Invalid binary package: '{full_path}', 
{e}\n",
+noiselevel=-1,
+)
 self.invalid_paths[mypkg] = [full_path]
 continue
 mycat = pkg_metadata.get("CATEGORY", "")
@@ -1073,10 +1080,11 @@ class binarytree:
 slot = pkg_metadata.get("SLOT", "")
 if not mycat or not mypf or not slot:
 # old-style or corrupt package
-writemsg(
-_("\n!!! Invalid binary package: '%s'\n") % 
full_path,
-noiselevel=-1,
-)
+if invalid_errors:
+writemsg(
+_("\n!!! Invalid binary package: '%s'\n") % 
full_path,
+noiselevel=-1,
+)
 missing_keys = []
 if not mycat:
 missing_keys.append("CATEGORY")
@@ -1087,18 +1095,20 @@ class binarytree:
 msg = []
 if missing_keys:
 missing_keys.sort()
+if invalid_errors:
+msg.append(
+_("Missing metadata key(s): %s.")
+% ", ".join(missing_keys)
+)
+if

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-10-04 Thread Zac Medico

commit: 2ccc08e3b629d3f56f028cd767ebd5ff1a8edaf1
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Oct  5 04:43:50 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Oct  5 05:18:58 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2ccc08e3

MergeProcess: Migrate to ForkProcess target parameter

Bug: https://bugs.gentoo.org/915099
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/_MergeProcess.py | 73 +++---
 1 file changed, 52 insertions(+), 21 deletions(-)

diff --git a/lib/portage/dbapi/_MergeProcess.py 
b/lib/portage/dbapi/_MergeProcess.py
index 7c018222fe..012435dce3 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -1,6 +1,7 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+import functools
 import io
 import multiprocessing
 import platform
@@ -73,6 +74,20 @@ class MergeProcess(ForkProcess):
 self.fd_pipes.setdefault(0, portage._get_stdin().fileno())
 
 self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
+self.target = functools.partial(
+self._target,
+lambda: self._counter,
+lambda: self._elog_reader_fd,
+lambda: self._dblink,
+self.infloc,
+self.mydbapi,
+self.myebuild,
+self.pkgloc,
+self.prev_mtimes,
+self.settings,
+self.unmerge,
+self.vartree.dbapi,
+)
 super()._start()
 
 def _lock_vdb(self):
@@ -197,13 +212,29 @@ class MergeProcess(ForkProcess):
 
 return pids
 
-def _run(self):
-os.close(self._elog_reader_fd)
-counter = self._counter
-mylink = self._dblink
-portage.output.havecolor = not no_color(self.settings)
+@staticmethod
+def _target(
+get_counter,
+get_elog_reader_fd,
+get_mylink,
+infloc,
+mydbapi,
+myebuild,
+pkgloc,
+prev_mtimes,
+settings,
+unmerge,
+vardb,
+):
+"""
+TODO: Make all arguments picklable for the multiprocessing spawn start 
method.
+"""
+os.close(get_elog_reader_fd())
+counter = get_counter()
+mylink = get_mylink()
+portage.output.havecolor = not no_color(settings)
 # Avoid wastful updates of the vdb cache.
-self.vartree.dbapi._flush_cache_enabled = False
+vardb._flush_cache_enabled = False
 
 # In this subprocess we don't want PORTAGE_BACKGROUND to
 # suppress stdout/stderr output since they are pipes. We
@@ -211,21 +242,21 @@ class MergeProcess(ForkProcess):
 # already be opened by the parent process, so we set the
 # "subprocess" value for use in conditional logging code
 # involving PORTAGE_LOG_FILE.
-if not self.unmerge:
+if not unmerge:
 # unmerge phases have separate logs
-if self.settings.get("PORTAGE_BACKGROUND") == "1":
-self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
+if settings.get("PORTAGE_BACKGROUND") == "1":
+settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
 else:
-self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
-self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
-self.settings["PORTAGE_BACKGROUND"] = "subprocess"
-self.settings.backup_changes("PORTAGE_BACKGROUND")
+settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
+settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
+settings["PORTAGE_BACKGROUND"] = "subprocess"
+settings.backup_changes("PORTAGE_BACKGROUND")
 
 rval = 1
-if self.unmerge:
+if unmerge:
 if not mylink.exists():
 rval = os.EX_OK
-elif mylink.unmerge(ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
+elif mylink.unmerge(ldpath_mtimes=prev_mtimes) == os.EX_OK:
 mylink.lockdb()
 try:
 mylink.delete()
@@ -234,11 +265,11 @@ class MergeProcess(ForkProcess):
 rval = os.EX_OK
 else:
 rval = mylink.merge(
-self.pkgloc,
-self.infloc,
-myebuild=self.myebuild,
-mydbapi=self.mydbapi,
-prev_mtimes=self.prev_mtimes,
+pkgloc,
+infloc,
+myebuild=myebuild,
+mydbapi=mydbapi,
+prev_mtimes=prev_mtimes,
 counter=counter,
 )
 return rval

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-10-03 Thread Zac Medico

commit: f45e2bb561f2f5d16bdf8f6cd31cc393d2794f92
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Oct  4 04:06:49 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Oct  4 04:11:14 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f45e2bb5

SyncfsProcess: Migrate to ForkProcess target parameter

Bug: https://bugs.gentoo.org/915099
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/_SyncfsProcess.py | 15 +++
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/_SyncfsProcess.py 
b/lib/portage/dbapi/_SyncfsProcess.py
index 6beeac8dd4..ddc2240071 100644
--- a/lib/portage/dbapi/_SyncfsProcess.py
+++ b/lib/portage/dbapi/_SyncfsProcess.py
@@ -1,6 +1,8 @@
-# Copyright 2012 Gentoo Foundation
+# Copyright 2012-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+import functools
+
 from portage import os
 from portage.util._ctypes import find_library, LoadLibrary
 from portage.util._async.ForkProcess import ForkProcess
@@ -16,6 +18,10 @@ class SyncfsProcess(ForkProcess):
 
 __slots__ = ("paths",)
 
+def _start(self):
+self.target = functools.partial(self._target, self._get_syncfs, 
self.paths)
+super()._start()
+
 @staticmethod
 def _get_syncfs():
 filename = find_library("c")
@@ -29,12 +35,13 @@ class SyncfsProcess(ForkProcess):
 
 return None
 
-def _run(self):
+@staticmethod
+def _target(get_syncfs, paths):
 syncfs_failed = False
-syncfs = self._get_syncfs()
+syncfs = get_syncfs()
 
 if syncfs is not None:
-for path in self.paths:
+for path in paths:
 try:
 fd = os.open(path, os.O_RDONLY)
 except OSError:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-09-26 Thread Sam James

commit: cec349d7cfcda634e3f76cd3ca0c1a89bc46f414
Author: hypersyd <70613804+siddhanthrathod  users  noreply  
github  com>
AuthorDate: Sat Sep  9 13:51:31 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Tue Sep 26 21:09:28 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cec349d7

bintree: populate invalid_paths list for eclean-pkg

Enable eclean-pkg to handle incomplete merges from binhost.

It's an extension for gentoolkit's eclean-pkg clean invalids feature to deal 
with
binpkgs created by the new binhost without any tar header or format due to 
incomplete merges.

This results in gpkg.py --> class gpkg --> method _get_tar_format returning 
null which raises
PortagePackageException and SignatureException in binpkg.py --> Class 
binarytree.

Add an input for invalid paths api is being added so the eclean-pkg can clean 
it up.

Invalids feature: 
https://github.com/gentoo/gentoolkit/commit/a16d0d4fbfb4614832c4b682b41284a9050af29f,
 
https://github.com/gentoo/portage/commit/71daef3ac877329a0479a72ba333a9c801a36bf3
Bug: https://bugs.gentoo.org/900224
Signed-off-by: Siddhanth Rathod  gmail.com>
Closes: https://github.com/gentoo/portage/pull/1091
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/bintree.py | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 42912b2eb5..580ce2f290 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1050,6 +1050,10 @@ class binarytree:
 else:
 binpkg_format = "gpkg"
 
+for ext in SUPPORTED_XPAK_EXTENSIONS + 
SUPPORTED_GPKG_EXTENSIONS:
+if myfile.endswith(ext):
+mypkg = myfile[: -len(ext)]
+break
 try:
 pkg_metadata = self._read_metadata(
 full_path,
@@ -1062,14 +1066,11 @@ class binarytree:
 f"!!! Invalid binary package: '{full_path}', 
{e}\n",
 noiselevel=-1,
 )
+self.invalid_paths[mypkg] = [full_path]
 continue
 mycat = pkg_metadata.get("CATEGORY", "")
 mypf = pkg_metadata.get("PF", "")
 slot = pkg_metadata.get("SLOT", "")
-for ext in SUPPORTED_XPAK_EXTENSIONS + 
SUPPORTED_GPKG_EXTENSIONS:
-if myfile.endswith(ext):
-mypkg = myfile[: -len(ext)]
-break
 if not mycat or not mypf or not slot:
 # old-style or corrupt package
 writemsg(

[gentoo-commits] proj/portage:master commit in: /, lib/portage/dbapi/

2023-09-26 Thread Sam James

commit: 5996c7bec3fe69e3ca69805777acdd8c5437b2ae
Author: gcarq  protonmail  com>
AuthorDate: Mon Sep  4 17:01:56 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Tue Sep 26 20:54:12 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5996c7be

vartree: Remove unused variables and parameters

Removes unused local variable cache_incomplete in aux_get()
and removes unused parameters for methods around counter_tick().

Signed-off-by: Michael Egger  protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1089
Signed-off-by: Sam James  gentoo.org>

 NEWS |  3 +++
 lib/portage/dbapi/vartree.py | 24 ++--
 2 files changed, 9 insertions(+), 18 deletions(-)

diff --git a/NEWS b/NEWS
index 340516a9b4..31a188e8f6 100644
--- a/NEWS
+++ b/NEWS
@@ -26,6 +26,9 @@ Bug fixes:
 
 * fetch: fix fetching of layout.conf when FEATURES=force-mirror (bug #877793).
 
+Cleanups:
+* vartree: Remove unused variables and parameters
+
 portage-3.0.51 (2023-08-20)
 --
 

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index d4b510082c..3f39e2b787 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -395,7 +395,7 @@ class vardbapi(dbapi):
 def cpv_inject(self, mycpv):
 "injects a real package into our on-disk database; assumes mycpv is 
valid and doesn't already exist"
 ensure_dirs(self.getpath(mycpv))
-counter = self.counter_tick(mycpv=mycpv)
+counter = self.counter_tick()
 # write local package counter so that emerge clean does the right thing
 write_atomic(self.getpath(mycpv, filename="COUNTER"), str(counter))
 
@@ -794,7 +794,6 @@ class vardbapi(dbapi):
 pull_me = cache_these.union(wants)
 mydata = {"_mtime_": mydir_mtime}
 cache_valid = False
-cache_incomplete = False
 cache_mtime = None
 metadata = None
 if pkg_data is not None:
@@ -1141,13 +1140,10 @@ class vardbapi(dbapi):
 log_path=settings.get("PORTAGE_LOG_FILE"),
 )
 
-def counter_tick(self, myroot=None, mycpv=None):
-"""
-@param myroot: ignored, self._eroot is used instead
-"""
-return self.counter_tick_core(incrementing=1, mycpv=mycpv)
+def counter_tick(self) -> int:
+return self.counter_tick_core(incrementing=1)
 
-def get_counter_tick_core(self, myroot=None, mycpv=None):
+def get_counter_tick_core(self) -> int:
 """
 Use this method to retrieve the counter instead
 of having to trust the value of a global counter
@@ -1165,10 +1161,7 @@ class vardbapi(dbapi):
 it also corresponds to the total number of
 installation actions that have occurred in
 the history of this package database.
-
-@param myroot: ignored, self._eroot is used instead
 """
-del myroot
 counter = -1
 try:
 with open(
@@ -1219,7 +1212,7 @@ class vardbapi(dbapi):
 
 return max_counter + 1
 
-def counter_tick_core(self, myroot=None, incrementing=1, mycpv=None):
+def counter_tick_core(self, incrementing: int = 1) -> int:
 """
 This method will grab the next COUNTER value and record it back
 to the global file. Note that every package install must have
@@ -1227,13 +1220,8 @@ class vardbapi(dbapi):
 into the same SLOT and in that case it's important that both
 packages have different COUNTER metadata.
 
-@param myroot: ignored, self._eroot is used instead
-@param mycpv: ignored
-@rtype: int
 @return: new counter value
 """
-myroot = None
-mycpv = None
 self.lock()
 try:
 counter = self.get_counter_tick_core() - 1
@@ -4974,7 +4962,7 @@ class dblink:
 
 # write local package counter for recording
 if counter is None:
-counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
+counter = self.vartree.dbapi.counter_tick()
 with open(
 _unicode_encode(
 os.path.join(self.dbtmpdir, "COUNTER"),

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-09-23 Thread Sam James

commit: 6293f7ba670fbfae5fd4bb6f4ba9fe3f822d19c3
Author: Sam James  gentoo  org>
AuthorDate: Sat Sep 23 22:49:29 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Sat Sep 23 22:49:49 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6293f7ba

vartree: fix syntax

Fixes: 1fc674667de01944269dfeccd70472facf83e7ba
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/vartree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 878914ef06..d4b510082c 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -2687,7 +2687,7 @@ class dblink:
 unmerge_orphans = "unmerge-orphans" in self.settings.features
 calc_prelink = "prelink-checksums" in self.settings.features
 
-pkgfiles = self.getcontents() if not pkgfiles
+pkgfiles = pkgfiles if pkgfiles else self.getcontents()
 if pkgfiles:
 self.updateprotect()
 mykeys = list(pkgfiles)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-09-23 Thread Sam James

commit: 1fc674667de01944269dfeccd70472facf83e7ba
Author: Sam James  gentoo  org>
AuthorDate: Sat Sep 23 22:38:23 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Sat Sep 23 22:38:31 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1fc67466

vartree: don't clobber pkgfiles

Fixes: 743722ccfa234dd9d4a54f7fbfb14cc2ddf6f0f4
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/vartree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 35a401b33c..878914ef06 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -2687,7 +2687,7 @@ class dblink:
 unmerge_orphans = "unmerge-orphans" in self.settings.features
 calc_prelink = "prelink-checksums" in self.settings.features
 
-pkgfiles = self.getcontents()
+pkgfiles = self.getcontents() if not pkgfiles
 if pkgfiles:
 self.updateprotect()
 mykeys = list(pkgfiles)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-09-23 Thread Sam James

commit: 743722ccfa234dd9d4a54f7fbfb14cc2ddf6f0f4
Author: Sam James  gentoo  org>
AuthorDate: Sat Sep 23 22:27:37 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Sat Sep 23 22:31:31 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=743722cc

vartree: drop 'Grabbing a set' useless noise

Prompted by genr8eofl asking about culling it.

Fascinatingly, both the message *and* people agreeing it's futile goes
back many years:
* bf01d6c02087c2b363617ce6ac208d09ea81de25 ('don't display useless messages 
when using --quiet')
* d9fc4acc572c6647a4f27b838d35d27d805d190e (svn migration)
* 4ba2baf8456c0eeda0df1bf52a1a3d712d5ac9b7 ('Portage-2.0.41 release. LOTS of 
bug fixes. Check the ChangeLog.', 2002)

This isn't useful for the user at all, so drop it.

Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/vartree.py | 5 +
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 08ea9234f6..35a401b33c 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -2660,10 +2660,6 @@ class dblink:
 ignored_unlink_errnos = self._ignored_unlink_errnos
 ignored_rmdir_errnos = self._ignored_rmdir_errnos
 
-if not pkgfiles:
-showMessage(_("No package files given... Grabbing a set.\n"))
-pkgfiles = self.getcontents()
-
 if others_in_slot is None:
 others_in_slot = []
 slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot
@@ -2691,6 +2687,7 @@ class dblink:
 unmerge_orphans = "unmerge-orphans" in self.settings.features
 calc_prelink = "prelink-checksums" in self.settings.features
 
+pkgfiles = self.getcontents()
 if pkgfiles:
 self.updateprotect()
 mykeys = list(pkgfiles)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-09-20 Thread Mike Gilbert

commit: 03489ca1880d4429c18cf7da2ed27ae65a21510b
Author: Andreas K. Hüttel  gentoo  org>
AuthorDate: Tue Sep 19 18:15:30 2023 +
Commit: Mike Gilbert  gentoo  org>
CommitDate: Wed Sep 20 18:02:20 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=03489ca1

Catch error when trust helper is not found

Closes: https://github.com/gentoo/portage/pull/1097
Signed-off-by: Andreas K. Hüttel  gentoo.org>
Signed-off-by: Mike Gilbert  gentoo.org>

 lib/portage/dbapi/bintree.py | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 9d7c45577b..42912b2eb5 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1245,7 +1245,17 @@ class binarytree:
 portage_trust_helper = self.settings.get("PORTAGE_TRUST_HELPER", "")
 if portage_trust_helper == "":
 return
-ret = subprocess.run(portage_trust_helper)
+try:
+ret = subprocess.run(portage_trust_helper)
+except FileNotFoundError:
+writemsg(
+_(
+"\n!!! Portage trust helper %s for binary packages not 
found\n!!! Continuing, but did you install app-portage/getuto?\n"
+)
+% portage_trust_helper,
+noiselevel=-1,
+)
+return
 ret.check_returncode()
 
 def _populate_remote(self, getbinpkg_refresh=True):

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-09-14 Thread Sam James

commit: b2c91af6fb2dfdd227caf9a2eff588f02d2bf9fc
Author: Oskari Pirhonen  gmail  com>
AuthorDate: Tue Sep  5 01:21:22 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Fri Sep 15 04:28:23 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2c91af6

vartree: keep build dir if postinst fails

Bug: https://bugs.gentoo.org/704866
Signed-off-by: Oskari Pirhonen  gmail.com>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/vartree.py | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 7d1bba712e..08ea9234f6 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -6070,8 +6070,11 @@ class dblink:
 ebuild_phase.wait()
 self._elog_process()
 
-if "noclean" not in self.settings.features and (
-retval == os.EX_OK or "fail-clean" in 
self.settings.features
+# Keep the build dir around if postinst fails (bug #704866)
+if (
+not self._postinst_failure
+and "noclean" not in self.settings.features
+and (retval == os.EX_OK or "fail-clean" in 
self.settings.features)
 ):
 if myebuild is None:
 myebuild = os.path.join(inforoot, self.pkg + ".ebuild")

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-07-28 Thread Sam James

commit: 7c0086fc0ab60cf0a726e3c88b11a21e147f3aa2
Author: Andrew Udvare  gmail  com>
AuthorDate: Sun Jul 16 21:42:58 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Sat Jul 29 03:57:45 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c0086fc

dbapi: add some typing

Signed-off-by: Andrew Udvare  gmail.com>
Closes: https://github.com/gentoo/portage/pull/1069
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/__init__.py | 27 +++
 lib/portage/dbapi/porttree.py | 34 --
 2 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 31453d149..428e4a48e 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -5,6 +5,7 @@ __all__ = ["dbapi"]
 
 import re
 import warnings
+from typing import Any, Dict, List, Optional, Sequence, Tuple
 
 import portage
 
@@ -29,7 +30,7 @@ from _emerge.Package import Package
 
 class dbapi:
 _category_re = re.compile(r"^\w[-.+\w]*$", re.UNICODE)
-_categories = None
+_categories: Optional[Tuple[str, ...]] = None
 _use_mutable = False
 _known_keys = frozenset(auxdbkeys)
 _pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository")
@@ -38,7 +39,7 @@ class dbapi:
 pass
 
 @property
-def categories(self):
+def categories(self) -> Tuple[str, ...]:
 """
 Use self.cp_all() to generate a category list. Mutable instances
 can delete the self._categories attribute in cases when the cached
@@ -52,11 +53,11 @@ class dbapi:
 def close_caches(self):
 pass
 
-def cp_list(self, cp, use_cache=1):
+def cp_list(self, cp: str, use_cache: int = 1) -> Any:
 raise NotImplementedError(self)
 
 @staticmethod
-def _cmp_cpv(cpv1, cpv2):
+def _cmp_cpv(cpv1, cpv2) -> int:
 result = vercmp(cpv1.version, cpv2.version)
 if result == 0 and cpv1.build_time is not None and cpv2.build_time is 
not None:
 result = (cpv1.build_time > cpv2.build_time) - (
@@ -65,7 +66,7 @@ class dbapi:
 return result
 
 @staticmethod
-def _cpv_sort_ascending(cpv_list):
+def _cpv_sort_ascending(cpv_list: Sequence[Any]) -> None:
 """
 Use this to sort self.cp_list() results in ascending
 order. It sorts in place and returns None.
@@ -76,7 +77,7 @@ class dbapi:
 # dict to map strings back to their original values.
 cpv_list.sort(key=cmp_sort_key(dbapi._cmp_cpv))
 
-def cpv_all(self):
+def cpv_all(self) -> List[str]:
 """Return all CPVs in the db
 Args:
 None
@@ -93,16 +94,18 @@ class dbapi:
 cpv_list.extend(self.cp_list(cp))
 return cpv_list
 
-def cp_all(self, sort=False):
+def cp_all(self, sort: bool = False) -> List[str]:
 """Implement this in a child class
 Args
 sort - return sorted results
 Returns:
 A list of strings 1 per CP in the datastore
 """
-return NotImplementedError
+raise NotImplementedError
 
-def aux_get(self, mycpv, mylist, myrepo=None):
+def aux_get(
+self, mycpv: str, mylist: str, myrepo: Optional[str] = None
+) -> List[str]:
 """Return the metadata keys in mylist for mycpv
 Args:
 mycpv - "sys-apps/foo-1.0"
@@ -114,7 +117,7 @@ class dbapi:
 """
 raise NotImplementedError
 
-def aux_update(self, cpv, metadata_updates):
+def aux_update(self, cpv: str, metadata_updates: Dict[str, Any]) -> None:
 """
 Args:
   cpv - "sys-apps/foo-1.0"
@@ -124,7 +127,7 @@ class dbapi:
 """
 raise NotImplementedError
 
-def match(self, origdep, use_cache=1):
+def match(self, origdep: str, use_cache: int = 1):
 """Given a dependency, try to find packages that match
 Args:
 origdep - Depend atom
@@ -138,7 +141,7 @@ class dbapi:
 self._iter_match(mydep, self.cp_list(mydep.cp, 
use_cache=use_cache))
 )
 
-def _iter_match(self, atom, cpv_iter):
+def _iter_match(self, atom: str, cpv_iter):
 cpv_iter = iter(match_from_list(atom, cpv_iter))
 if atom.repo:
 cpv_iter = self._iter_match_repo(atom, cpv_iter)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 3c38e99d4..c47b66bda 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -49,6 +49,7 @@ import functools
 
 import collections
 from collections import OrderedDict
+from typing import List, Optional, Sequence, Type, Tuple, Union
 from urllib.parse import urlparse
 
 
@@ -435,7 +436,9 @@ class portdbapi(dbapi):
 return license_path
 return None
 
-def findname(self, mycpv, mytree=None, myrepo=None):
+def findname(
+self, mycpv: str, mytree:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/, man/

2023-05-25 Thread Sam James

commit: da70f89cd88968501ca07cfaca6d73665d6e767a
Author: Ulrich Müller  gentoo  org>
AuthorDate: Tue May 23 09:08:36 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Fri May 26 01:33:06 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=da70f89c

Drop ignore-mtime from features

The ignore-mtime feature violates PMS:
https://projects.gentoo.org/pms/8/pms.html#x1-14500013.3.2

Allowing the package manager to arbitrarily modify timestamps of
installed files will lead to problems with several languages:

  - Lisp (*.fasl must be newer than corresponding *.lisp)
  - Emacs (*.elc must not be older than *.el)
  - ghdl/VHDL (libraries check their mtimes)
  - Python (*.pyc vs *.py, not sure if this one is still an issue)

If the PM does not preserve timestamps, there is no reasonable way to
work around this on the ebuild level. (Some horrible hacks existed in
the past, see for example impl-*-timestamp-hack in previous versions
of common-lisp-common*.eclass which used to overwrite its installed
files in pkg_postinst.)

This partially reverts commit 89703c688868c9eb8cd6115cb42ff92f0b9668b8.

Closes: https://bugs.gentoo.org/906978
See-also: https://bugs.gentoo.org/264130
Signed-off-by: Ulrich Müller  gentoo.org>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/const.py |  1 -
 lib/portage/dbapi/vartree.py | 14 ++
 man/make.conf.5  |  6 --
 3 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/lib/portage/const.py b/lib/portage/const.py
index cb6796164..10a208ceb 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -159,7 +159,6 @@ SUPPORTED_FEATURES = frozenset(
 "getbinpkg",
 "gpg-keepalive",
 "icecream",
-"ignore-mtime",
 "installsources",
 "ipc-sandbox",
 "keeptemp",

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 441e74661..dac350880 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -5422,7 +5422,6 @@ class dblink:
 srcroot = normalize_path(srcroot).rstrip(sep) + sep
 destroot = normalize_path(destroot).rstrip(sep) + sep
 calc_prelink = "prelink-checksums" in self.settings.features
-ignore_mtime = "ignore-mtime" in self.settings.features
 
 protect_if_modified = (
 "config-protect-if-modified" in self.settings.features
@@ -5831,13 +5830,12 @@ class dblink:
 hardlink_candidates.append(mydest)
 zing = ">>>"
 else:
-if not ignore_mtime:
-mymtime = thismtime if thismtime is not None else 
mymtime
-try:
-os.utime(mydest, ns=(mymtime, mymtime))
-except OSError:
-# utime can fail here with EPERM
-pass
+mymtime = thismtime if thismtime is not None else 
mymtime
+try:
+os.utime(mydest, ns=(mymtime, mymtime))
+except OSError:
+# utime can fail here with EPERM
+pass
 zing = "==="
 
 try:

diff --git a/man/make.conf.5 b/man/make.conf.5
index ed5b7d7fc..85ee88c05 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -533,12 +533,6 @@ If your GPG is auto unlocked on login, you do not need 
this.
 .B icecream
 Enable portage support for the icecream package.
 .TP
-.B ignore\-mtime
-Do not update mtime if the target file is equal. This can be useful on some
-filesystems to better utilize features like snapshots and data deduplication,
-however this violates the preservation of file modification times as stipulated
-in the PMS.
-.TP
 .B installsources
 Install source code into /usr/src/debug/${CATEGORY}/${PF} (also see
 \fBsplitdebug\fR). This feature works only if debugedit is installed, CFLAGS

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-05-22 Thread Sam James

commit: 6da3e0fd0ddf551cf39b6ac329ee51051a22
Author: gcarq  protonmail  com>
AuthorDate: Mon Mar 27 13:47:52 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Tue May 23 00:22:09 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6da3e0fd

mergeme: Put xattr comparison logic behind xattr feature flag

Signed-off-by: gcarq  protonmail.com>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/vartree.py | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 676d4aa05..739b47f7f 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -6267,9 +6267,10 @@ class dblink:
 if mydmode is None or not stat.S_ISREG(mydmode) or mymode != mydmode:
 return True
 
-excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
-if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs):
-return True
+if "xattr" in self.settings.features:
+excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
+if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs):
+return True
 
 return not filecmp.cmp(mysrc, mydest, shallow=False)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-05-22 Thread Sam James

commit: a87be47f7d3245050da43d7c3ab4760d47e9fac5
Author: gcarq  protonmail  com>
AuthorDate: Tue Feb 21 00:04:26 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Tue May 23 00:22:09 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a87be47f

mergeme: Don't overwrite files if the content matches

Uses filecmp.cmp(shallow=False) to compare file contents and
doesn't replace them if they are equal. This results in less disk
churn and helps to keep filesystem snapshots as small as possible.

Closes: https://bugs.gentoo.org/722270
Signed-off-by: gcarq  protonmail.com>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/vartree.py | 47 +---
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index a9e332a74..327b72bed 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -3,6 +3,8 @@
 
 __all__ = ["vardbapi", "vartree", "dblink"] + ["write_contents", 
"tar_contents"]
 
+import filecmp
+
 import portage
 
 portage.proxy.lazyimport.lazyimport(
@@ -5800,28 +5802,33 @@ class dblink:
 # whether config protection or not, we merge the new file the
 # same way.  Unless moveme=0 (blocking directory)
 if moveme:
-# Create hardlinks only for source files that already exist
-# as hardlinks (having identical st_dev and st_ino).
-hardlink_key = (mystat.st_dev, mystat.st_ino)
+# only replace the existing file if it differs, see #722270
+already_merged = os.path.exists(mydest)
+if already_merged and filecmp.cmp(mysrc, mydest, 
shallow=False):
+zing = "==="
+else:
+# Create hardlinks only for source files that already 
exist
+# as hardlinks (having identical st_dev and st_ino).
+hardlink_key = (mystat.st_dev, mystat.st_ino)
 
-hardlink_candidates = 
self._hardlink_merge_map.get(hardlink_key)
-if hardlink_candidates is None:
-hardlink_candidates = []
-self._hardlink_merge_map[hardlink_key] = 
hardlink_candidates
+hardlink_candidates = 
self._hardlink_merge_map.get(hardlink_key)
+if hardlink_candidates is None:
+hardlink_candidates = []
+self._hardlink_merge_map[hardlink_key] = 
hardlink_candidates
 
-mymtime = movefile(
-mysrc,
-mydest,
-newmtime=thismtime,
-sstat=mystat,
-mysettings=self.settings,
-hardlink_candidates=hardlink_candidates,
-encoding=_encodings["merge"],
-)
-if mymtime is None:
-return 1
-hardlink_candidates.append(mydest)
-zing = ">>>"
+mymtime = movefile(
+mysrc,
+mydest,
+newmtime=thismtime,
+sstat=mystat,
+mysettings=self.settings,
+hardlink_candidates=hardlink_candidates,
+encoding=_encodings["merge"],
+)
+if mymtime is None:
+return 1
+hardlink_candidates.append(mydest)
+zing = ">>>"
 
 try:
 self._merged_path(mydest, os.lstat(mydest))

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2023-05-22 Thread Sam James

commit: 35b5e4d71ebb5c7408dba7dc27cff0c22cad1562
Author: gcarq  protonmail  com>
AuthorDate: Mon Mar 27 13:31:44 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Tue May 23 00:22:09 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=35b5e4d7

mergeme: Rely on mydmode instead of calling os.path.exists again

Signed-off-by: gcarq  protonmail.com>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/vartree.py | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 317cf327a..676d4aa05 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -5544,6 +5544,8 @@ class dblink:
 destmd5,
 mydest_link,
 )
+if protected and moveme:
+mydmode = None
 
 zing = "!!!"
 if not moveme:
@@ -5584,6 +5586,7 @@ class dblink:
 msg.append("")
 self._eerror("preinst", msg)
 mydest = newdest
+mydmode = None
 
 # if secondhand is None it means we're operating in "force" 
mode and should not create a second hand.
 if (secondhand is not None) and (not os.path.exists(myrealto)):
@@ -5797,6 +5800,7 @@ class dblink:
 msg.append("")
 self._eerror("preinst", msg)
 mydest = newdest
+mydmode = None
 
 # whether config protection or not, we merge the new file the
 # same way.  Unless moveme=0 (blocking directory)
@@ -6260,10 +6264,7 @@ class dblink:
 Takes file mode and extended attributes into account.
 Should only be used for regular files.
 """
-if not os.path.exists(mydest):
-return True
-
-if mymode != mydmode:
+if mydmode is None or not stat.S_ISREG(mydmode) or mymode != mydmode:
 return True
 
 excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")

[gentoo-commits] proj/portage:master commit in: /, lib/portage/dbapi/

2023-04-07 Thread Sam James

commit: 2569a1a1d889a76af80ce24a005d54269df7d2e2
Author: Sheng Yu  protonmail  com>
AuthorDate: Fri Apr  7 04:50:49 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Fri Apr  7 09:49:11 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2569a1a1

dbapi: Catch invalid binpkg error during pkgmove

Bug: https://bugs.gentoo.org/877271
Bug: https://bugs.gentoo.org/903917
Bug: https://bugs.gentoo.org/903926
Signed-off-by: Sheng Yu  protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1022
Signed-off-by: Sam James  gentoo.org>

 NEWS  | 3 +++
 lib/portage/dbapi/__init__.py | 8 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 506e673e1..1d05f1de2 100644
--- a/NEWS
+++ b/NEWS
@@ -60,6 +60,9 @@ Features:
   for a time for compatibility.
 
 Bug fixes:
+* dbapi: Handle mismatched binpkg structure during pkgmoves (bug #877271,
+  bug #903917, bug #903926).
+
 * tests: util/test_shelve: fix test failure if the backend for the shelve 
module
   does not create the shelve db using the literal filename.
 

diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 366a6c170..31453d149 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -4,6 +4,7 @@
 __all__ = ["dbapi"]
 
 import re
+import warnings
 
 import portage
 
@@ -21,7 +22,7 @@ from portage.const import MERGING_IDENTIFIER
 from portage import os
 from portage import auxdbkeys
 from portage.eapi import _get_eapi_attrs
-from portage.exception import InvalidData
+from portage.exception import InvalidBinaryPackageFormat, InvalidData
 from portage.localization import _
 from _emerge.Package import Package
 
@@ -410,7 +411,10 @@ class dbapi:
 updates_list, metadata, parent=pkg
 )
 if metadata_updates:
-aux_update(cpv, metadata_updates)
+try:
+aux_update(cpv, metadata_updates)
+except InvalidBinaryPackageFormat as e:
+warnings.warn(e)
 if onUpdate:
 onUpdate(maxval, i + 1)
 if onProgress:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/resolver/, /, lib/portage/_emirrordist/, ...

2023-03-20 Thread Sam James

commit: 0431aa7db28e69af309a8175ec9e13a23439e2fa
Author: Siddhanth Rathod  gmail  com>
AuthorDate: Sat Mar  4 19:29:28 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Tue Mar 21 02:35:55 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0431aa7d

emerge: respect NO_COLOR

Bug: https://bugs.gentoo.org/898224
Signed-off-by: Siddhanth Rathod  gmail.com>
Closes: https://github.com/gentoo/portage/pull/1003
Signed-off-by: Sam James  gentoo.org>

 NEWS   |  7 ++-
 bin/ebuild |  7 +++
 bin/egencache  |  9 +++--
 bin/etc-update |  1 +
 bin/isolated-functions.sh  | 11 +--
 bin/portageq   |  6 ++
 bin/save-ebuild-env.sh |  2 +-
 lib/_emerge/AbstractEbuildProcess.py   |  7 +++
 lib/_emerge/actions.py | 12 ++--
 lib/portage/_emirrordist/main.py   |  2 +-
 lib/portage/dbapi/_MergeProcess.py |  5 ++---
 lib/portage/package/ebuild/_config/special_env_vars.py |  2 ++
 lib/portage/package/ebuild/_ipc/QueryCommand.py|  6 ++
 lib/portage/tests/__init__.py  |  3 ++-
 lib/portage/tests/resolver/ResolverPlayground.py   |  4 ++--
 lib/portage/tests/runTests.py  |  2 +-
 lib/portage/util/__init__.py   | 11 +++
 17 files changed, 57 insertions(+), 40 deletions(-)

diff --git a/NEWS b/NEWS
index 03e262c69..01209c9fb 100644
--- a/NEWS
+++ b/NEWS
@@ -48,7 +48,12 @@ Cleanups:
   in terms of priority for providers.
 
 Features:
-* TODO
+* Respect the NO_COLOR environment variable for disabling color.
+
+  The new quasi-standard for disabling ANSI color is to check for the NO_COLOR
+  environment variable and disable color when the variable has a nonempty 
value.
+  See bug #898224. Portage previously used NOCOLOR. It continues to support 
NOCOLOR
+  for a time for compatibility.
 
 Bug fixes:
 * TODO

diff --git a/bin/ebuild b/bin/ebuild
index 8d1908795..8fccbea89 100755
--- a/bin/ebuild
+++ b/bin/ebuild
@@ -67,7 +67,6 @@ try:
 from _emerge.RootConfig import RootConfig
 
 portage.process.sanitize_fds()
-
 description = "See the ebuild(1) man page for more info"
 usage = "Usage: ebuild   [command] ..."
 parser = argparse.ArgumentParser(description=description, usage=usage)
@@ -128,14 +127,14 @@ try:
 
 if not opts.color == "y" and (
 opts.color == "n"
-or portage.settings.get("NOCOLOR") in ("yes", "true")
+or portage.util.no_color(portage.settings)
 or portage.settings.get("TERM") == "dumb"
 or not sys.stdout.isatty()
 ):
 portage.output.nocolor()
 portage.settings.unlock()
-portage.settings["NOCOLOR"] = "true"
-portage.settings.backup_changes("NOCOLOR")
+portage.settings["NO_COLOR"] = "true"
+portage.settings.backup_changes("NO_COLOR")
 portage.settings.lock()
 
 apply_priorities(portage.settings)

diff --git a/bin/egencache b/bin/egencache
index 1a6118bde..671df3014 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -67,7 +67,7 @@ try:
 from portage.package.ebuild._parallel_manifest.ManifestScheduler import (
 ManifestScheduler,
 )
-from portage.util import cmp_sort_key, writemsg_level
+from portage.util import cmp_sort_key, writemsg_level, no_color
 from portage.util._async.AsyncFunction import AsyncFunction
 from portage.util._async.run_main_scheduler import run_main_scheduler
 from portage.util._async.TaskScheduler import TaskScheduler
@@ -1105,12 +1105,9 @@ try:
 if "PATH" in os.environ:
 env["PATH"] = os.environ["PATH"]
 
-if not sys.stdout.isatty() or os.environ.get("NOCOLOR", "").lower() in 
(
-"yes",
-"true",
-):
+if not sys.stdout.isatty() or no_color(os.environ):
 portage.output.nocolor()
-env["NOCOLOR"] = "true"
+env["NO_COLOR"] = "true"
 
 parser, options, atoms = parse_args(args)
 

diff --git a/bin/etc-update b/bin/etc-update
index 59e709168..14bd80b84 100755
--- a/bin/etc-update
+++ b/bin/etc-update
@@ -798,6 +798,7 @@ portage_vars=(
EROOT
USERLAND
NOCOLOR
+   NO_COLOR
 )
 
 if type -P portageq > /dev/null; then

diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh
index 882789132..06be030fb 100644
--- a/bin/isolated-functions.sh
+++ b/bin/isolated-functions.sh
@@ -434,14 +434,21 @@ RC_INDENTATION=''
 RC_DEFAULT_INDENT=2
 RC_DOT_PATTERN=''
 
-case "${NOCOLOR:-false}" in
+
+
+if [[ -z ${NO_COLOR} ]] ; then
+   case ${NOCOLOR:-false} in
yes|true)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/update/

2023-02-16 Thread Sam James

commit: c00dbfb9d8e3c494173200436ec4c5bc2689707f
Author: Sheng Yu  protonmail  com>
AuthorDate: Fri Feb 10 07:55:26 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Fri Feb 17 01:23:14 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c00dbfb9

Ignore update GPG signed binpkg

User cannot update them without key.

Signed-off-by: Sheng Yu  protonmail.com>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/bintree.py  | 8 
 lib/portage/tests/update/test_move_ent.py | 5 -
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 23bc41d44..5f58c548d 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -674,6 +674,14 @@ class binarytree:
 elif binpkg_format == "gpkg":
 mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, binpkg_path)
 mydata = mybinpkg.get_metadata()
+if mybinpkg.signature_exist:
+writemsg(
+colorize(
+"WARN",
+f"Binpkg update ignored for signed package: 
{binpkg_path}",
+)
+)
+continue
 decode_metadata_name = True
 else:
 continue

diff --git a/lib/portage/tests/update/test_move_ent.py 
b/lib/portage/tests/update/test_move_ent.py
index 562d7b107..22d0c8feb 100644
--- a/lib/portage/tests/update/test_move_ent.py
+++ b/lib/portage/tests/update/test_move_ent.py
@@ -58,7 +58,10 @@ class MoveEntTestCase(TestCase):
 ebuilds=ebuilds,
 installed=installed,
 user_config={
-"make.conf": (f'BINPKG_FORMAT="{binpkg_format}"',),
+"make.conf": (
+f'BINPKG_FORMAT="{binpkg_format}"',
+'FEATURES="-binpkg-signing"',
+),
 },
 )

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2022-12-20 Thread Sam James

commit: 2f020ae6aca04a0db68b7ceb5413e2b815d2169d
Author: Sam James  gentoo  org>
AuthorDate: Wed Dec 21 01:30:22 2022 +
Commit: Sam James  gentoo  org>
CommitDate: Wed Dec 21 01:30:55 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2f020ae6

Revert "porttree: skip metadata verification if repository is immutable (not 
volatile)"

This reverts commit f05140beb2dd9a3577ff2042941cb3e4fbb1df31.

I didn't mean to push this one yet, which we're going to handle
separately.

Bug: https://bugs.gentoo.org/528394
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/porttree.py | 7 ++-
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 448a7f300..15e1fd6ff 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -614,8 +614,7 @@ class portdbapi(dbapi):
 if ro_auxdb is not None:
 auxdbs.append(ro_auxdb)
 auxdbs.append(self.auxdb[repo_path])
-repo = self.repositories.get_repo_for_location(repo_path)
-eclass_db = repo.eclass_db
+eclass_db = 
self.repositories.get_repo_for_location(repo_path).eclass_db
 
 for auxdb in auxdbs:
 try:
@@ -638,9 +637,7 @@ class portdbapi(dbapi):
 # EAPI from _parse_eapi_ebuild_head, we disregard cache entries
 # for unsupported EAPIs.
 continue
-if not repo.volatile or auxdb.validate_entry(
-metadata, ebuild_hash, eclass_db
-):
+if auxdb.validate_entry(metadata, ebuild_hash, eclass_db):
 break
 else:
 metadata = None

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2022-12-20 Thread Sam James

commit: f05140beb2dd9a3577ff2042941cb3e4fbb1df31
Author: Sam James  gentoo  org>
AuthorDate: Sat Dec 17 05:15:27 2022 +
Commit: Sam James  gentoo  org>
CommitDate: Wed Dec 21 01:28:02 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f05140be

porttree: skip metadata verification if repository is immutable (not volatile)

With Python 3.11, I get:
- ~232695 total syscalls before (38268 read calls)
- ~203656 total syscalls after (27381 read calls)

It's a cheap improvement for simply promising to not touch
ebuilds in a repository.

Bug: https://bugs.gentoo.org/528394
See: a72a01746638debe472496bd8fc661992a6ba08b
See: 5c7bf4eb09f644813a6f017ffd91665664142560
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/porttree.py | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 15e1fd6ff..448a7f300 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -614,7 +614,8 @@ class portdbapi(dbapi):
 if ro_auxdb is not None:
 auxdbs.append(ro_auxdb)
 auxdbs.append(self.auxdb[repo_path])
-eclass_db = 
self.repositories.get_repo_for_location(repo_path).eclass_db
+repo = self.repositories.get_repo_for_location(repo_path)
+eclass_db = repo.eclass_db
 
 for auxdb in auxdbs:
 try:
@@ -637,7 +638,9 @@ class portdbapi(dbapi):
 # EAPI from _parse_eapi_ebuild_head, we disregard cache entries
 # for unsupported EAPIs.
 continue
-if auxdb.validate_entry(metadata, ebuild_hash, eclass_db):
+if not repo.volatile or auxdb.validate_entry(
+metadata, ebuild_hash, eclass_db
+):
 break
 else:
 metadata = None

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, /

2022-11-30 Thread Sam James

commit: 66f64d08ffb13ca24cfc11584fe667b1c391fbe8
Author: Sam James  gentoo  org>
AuthorDate: Thu Dec  1 06:31:53 2022 +
Commit: Sam James  gentoo  org>
CommitDate: Thu Dec  1 07:08:02 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=66f64d08

vartree: use ewarn if merging despite collisions

Use ewarn, not eerror, if merge is going ahead despite collisions.

Collisions aren't always fatal - say if FEATURES="protect-owned unmerge-orphans"
and the collision is over an orphaned file.

This changes the colors used once we discover that all relevant files
are orphaned to be warning rather than an error, to make it less scary
and distinguish from other situations.

Closes: https://github.com/gentoo/portage/pull/952
Signed-off-by: Sam James  gentoo.org>

 NEWS |  4 +++-
 lib/portage/dbapi/vartree.py | 18 +++---
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/NEWS b/NEWS
index d64fcac91..a907526aa 100644
--- a/NEWS
+++ b/NEWS
@@ -2,7 +2,9 @@ portage-3.0.41 (UNRELEASED)
 --
 
 Features:
-* TODO
+* vartree: On collisions which are non-fatal (e.g. orphaned files when using
+  FEATURES="protect-owned"), Portage now has friendlier output
+  by using ewarn rather than eerror.
 
 Bug fixes:
 * TODO

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index a95d60691..139424c0a 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -4187,6 +4187,9 @@ class dblink:
 def _eqawarn(self, phase, lines):
 self._elog("eqawarn", phase, lines)
 
+def _ewarn(self, phase, lines):
+self._elog("ewarn", phase, lines)
+
 def _eerror(self, phase, lines):
 self._elog("eerror", phase, lines)
 
@@ -4388,6 +4391,9 @@ class dblink:
 ],
 )
 
+def ewarn(lines):
+self._ewarn("preinst", lines)
+
 def eerror(lines):
 self._eerror("preinst", lines)
 
@@ -4884,6 +4890,10 @@ class dblink:
 finally:
 self.unlockdb()
 
+collision_message_type = ewarn
+if collision_protect or protect_owned and owners:
+collision_message_type = eerror
+
 for pkg, owned_files in owners.items():
 msg = []
 msg.append(pkg_info_strs[pkg.mycpv])
@@ -4892,10 +4902,10 @@ class dblink:
 "\t%s" % os.path.join(destroot, 
f.lstrip(os.path.sep))
 )
 msg.append("")
-eerror(msg)
+collision_message_type(msg)
 
 if not owners:
-eerror(
+collision_message_type(
 [_("None of the installed" " packages claim the 
file(s)."), ""]
 )
 
@@ -4935,10 +4945,12 @@ class dblink:
 " If necessary, refer to your elog "
 "messages for the whole content of the above message."
 )
-eerror(wrap(msg, 70))
 
 if abort:
+eerror(wrap(msg, 70))
 return 1
+else:
+ewarn(wrap(msg, 70))
 
 # The merge process may move files out of the image directory,
 # which causes invalidation of the .installed flag.

[gentoo-commits] proj/portage:master commit in: /, lib/portage/dbapi/

2022-11-30 Thread Sam James

commit: d9699fa0b09a1d5a397edf0bb25c7662ac88e09d
Author: Sheng Yu  protonmail  com>
AuthorDate: Wed Nov 30 18:13:29 2022 +
Commit: Sam James  gentoo  org>
CommitDate: Wed Nov 30 22:28:58 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d9699fa0

Properly assign filenames for remote packages

Signed-off-by: Sheng Yu  protonmail.com>
Closes: https://github.com/gentoo/portage/pull/950
Signed-off-by: Sam James  gentoo.org>

 NEWS | 2 ++
 lib/portage/dbapi/bintree.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 923b95a43..1cb2b2468 100644
--- a/NEWS
+++ b/NEWS
@@ -49,6 +49,8 @@ Bug fixes:
 * env-update: Also generate PATH definition in systemd user environment file
   /etc/environment.d/10-gentoo-env.conf
 
+* bintree: Properly assign filenames for remote binpkgs.
+
 * --disable-static is only passed for libtool-enabled configure scripts in 
EAPI 8.
   This avoids annoying warnings when a configure script has a flag such as
   --disable-static_link that would then trigger a QA warning (bug #814380).

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index d7c18e2e9..baf001c8f 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -2031,7 +2031,7 @@ class binarytree:
 filename = None
 build_id = None
 if allocate_new:
-filename, build_id = self._allocate_filename(cpv)
+filename, build_id = self._allocate_filename(cpv, 
remote_binpkg_format)
 elif self._is_specific_instance(cpv):
 instance_key = self.dbapi._instance_key(cpv)
 path = self._pkg_paths.get(instance_key)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2022-11-08 Thread Sam James

commit: a7dd39c1ae4a5ea4e3252ae8129fbd671c95d5f7
Author: Sheng Yu  protonmail  com>
AuthorDate: Tue Nov  8 22:52:46 2022 +
Commit: Sam James  gentoo  org>
CommitDate: Tue Nov  8 23:07:46 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7dd39c1

Do not drop default PATH in the package index (avoid Packages regeneration)

The new Packages index (introduced in 445f10f4214c673f8fe0a9cc518c12767be4f159)
needs PATH but it got dropped later on when processing, so we would try
to rebuild the index on every emerge call.

This fixes regenerating Packages loop.

(Note that this didn't affect a released version.)

Fixes: 445f10f4214c673f8fe0a9cc518c12767be4f159
Bug: https://bugs.gentoo.org/877357
Bug: https://bugs.gentoo.org/877419
Signed-off-by: Sheng Yu  protonmail.com>
Closes: https://github.com/gentoo/portage/pull/934
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/bintree.py | 21 +++--
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 771abedd5..d7c18e2e9 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1000,20 +1000,9 @@ class binarytree:
 pkg_paths[instance_key] = mypath
 # update the path if the package has been moved
 oldpath = d.get("PATH")
-if oldpath and oldpath != mypath:
-update_pkgindex = True
-# Omit PATH if it is the default path for
-# the current Packages format version.
-if (mypath != mycpv + ".tbz2") and (
-mypath != mycpv + ".gpkg.tar"
-):
+if oldpath != mypath:
 d["PATH"] = mypath
-if not oldpath:
-update_pkgindex = True
-else:
-d.pop("PATH", None)
-if oldpath:
-update_pkgindex = True
+update_pkgindex = True
 self.dbapi.cpv_inject(mycpv)
 continue
 if not os.access(full_path, os.R_OK):
@@ -1218,11 +1207,7 @@ class binarytree:
 self.dbapi.cpv_remove(mycpv)
 del pkg_paths[_instance_key(mycpv)]
 
-# record location if it's non-default
-if (mypath != mycpv + ".tbz2") and (mypath != mycpv + 
".gpkg.tar"):
-d["PATH"] = mypath
-else:
-d.pop("PATH", None)
+d["PATH"] = mypath
 metadata[_instance_key(mycpv)] = d
 
 if reindex:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/emerge/, lib/portage/tests/dbapi/

2022-09-25 Thread Mike Gilbert

commit: 31c04f4b5b014a6a80ad4d1c2c8574106bab2c56
Author: David Palao  gmail  com>
AuthorDate: Fri Sep 16 15:08:41 2022 +
Commit: Mike Gilbert  gentoo  org>
CommitDate: Sun Sep 25 19:10:34 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=31c04f4b

improvement(bintree)!: binarytree.populate has new default 
getbinpkg_refresh=False

This, together with a previous commit, fixes Bug #864259:

sys-apps/portage: fetches binhost's 'Packages' file multiple times

Bug: https://bugs.gentoo.org/864259
Signed-off-by: David Palao  gmail.com>
Signed-off-by: Mike Gilbert  gentoo.org>

 lib/portage/dbapi/bintree.py |  2 +-
 lib/portage/tests/dbapi/test_bintree.py  | 17 +
 lib/portage/tests/emerge/test_actions.py |  5 +
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index cea9378d5..c8bb2c88e 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -807,7 +807,7 @@ class binarytree:
 except PortageException:
 pass
 
-def populate(self, getbinpkgs=False, getbinpkg_refresh=True, add_repos=()):
+def populate(self, getbinpkgs=False, getbinpkg_refresh=False, 
add_repos=()):
 """
 Populates the binarytree with package metadata.
 

diff --git a/lib/portage/tests/dbapi/test_bintree.py 
b/lib/portage/tests/dbapi/test_bintree.py
index 881d8ff48..4b4982a54 100644
--- a/lib/portage/tests/dbapi/test_bintree.py
+++ b/lib/portage/tests/dbapi/test_bintree.py
@@ -136,3 +136,20 @@ class BinarytreeTestCase(TestCase):
 ),
 noiselevel=-1,
 )
+
+@patch("portage.dbapi.bintree.BinRepoConfigLoader")
+@patch("portage.dbapi.bintree.binarytree._populate_remote")
+@patch("portage.dbapi.bintree.binarytree._populate_local")
+def test_default_getbinpkg_refresh_in_populate(
+self, ppopulate_local, ppopulate_remote, pBinRepoConfigLoader
+):
+"""Bug #864259
+This test fixes the bug. It requires that
+``_emerge.actions.run_action`` calls ``binarytree.populate``
+explicitly with ``getbinpkg_refresh=True``
+"""
+settings = MagicMock()
+settings.__getitem__.return_value = "/some/path"
+bt = binarytree(pkgdir="/tmp", settings=settings)
+bt.populate(getbinpkgs=True)
+ppopulate_remote.assert_called_once_with(getbinpkg_refresh=False)

diff --git a/lib/portage/tests/emerge/test_actions.py 
b/lib/portage/tests/emerge/test_actions.py
index 014d39dd9..ee59cef5e 100644
--- a/lib/portage/tests/emerge/test_actions.py
+++ b/lib/portage/tests/emerge/test_actions.py
@@ -30,6 +30,11 @@ class RunActionTestCase(TestCase):
 papply,
 padjust,
 profile_ckeck):
+"""Ensure that ``binarytree.populate`` API is correctly used.
+The point of this test is to ensure that the ``populate`` method
+is called as expected: since it is the first time that ``populate``
+is called, it must use ``getbinpkg_refresh=True``.
+"""
 config = MagicMock()
 config.action = None
 config.opts = {"--quiet": True, "--usepkg": True}

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2022-09-25 Thread Mike Gilbert

commit: a83507be7ce04d3ac421f9cbe8b63816809b0f4e
Author: David Palao  gmail  com>
AuthorDate: Fri Sep 23 13:42:33 2022 +
Commit: Mike Gilbert  gentoo  org>
CommitDate: Sun Sep 25 19:07:52 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a83507be

chore(bintree): removed useless ``if True:``

This commit de-indents a large block.

Signed-off-by: David Palao  gmail.com>
Signed-off-by: Mike Gilbert  gentoo.org>

 lib/portage/dbapi/bintree.py | 263 +--
 1 file changed, 131 insertions(+), 132 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 7f5dc051c..28b3c481b 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -473,143 +473,142 @@ class binarytree:
 stacklevel=2,
 )
 
-if True:
-self.pkgdir = normalize_path(pkgdir)
-# NOTE: Event if binpkg-multi-instance is disabled, it's
-# still possible to access a PKGDIR which uses the
-# binpkg-multi-instance layout (or mixed layout).
-self._multi_instance = "binpkg-multi-instance" in settings.features
-if self._multi_instance:
-self._allocate_filename = self._allocate_filename_multi
-self.dbapi = bindbapi(self, settings=settings)
-self.update_ents = self.dbapi.update_ents
-self.move_slot_ent = self.dbapi.move_slot_ent
-self.populated = 0
-self.tree = {}
-self._binrepos_conf = None
-self._remote_has_index = False
-self._remotepkgs = None  # remote metadata indexed by cpv
-self._additional_pkgs = {}
-self.invalids = []
-self.settings = settings
-self._pkg_paths = {}
-self._populating = False
-self._all_directory = os.path.isdir(os.path.join(self.pkgdir, 
"All"))
-self._pkgindex_version = 0
-self._pkgindex_hashes = ["MD5", "SHA1"]
-self._pkgindex_file = os.path.join(self.pkgdir, "Packages")
-self._pkgindex_keys = self.dbapi._aux_cache_keys.copy()
-self._pkgindex_keys.update(["CPV", "SIZE"])
-self._pkgindex_aux_keys = [
-"BASE_URI",
-"BDEPEND",
+self.pkgdir = normalize_path(pkgdir)
+# NOTE: Event if binpkg-multi-instance is disabled, it's
+# still possible to access a PKGDIR which uses the
+# binpkg-multi-instance layout (or mixed layout).
+self._multi_instance = "binpkg-multi-instance" in settings.features
+if self._multi_instance:
+self._allocate_filename = self._allocate_filename_multi
+self.dbapi = bindbapi(self, settings=settings)
+self.update_ents = self.dbapi.update_ents
+self.move_slot_ent = self.dbapi.move_slot_ent
+self.populated = 0
+self.tree = {}
+self._binrepos_conf = None
+self._remote_has_index = False
+self._remotepkgs = None  # remote metadata indexed by cpv
+self._additional_pkgs = {}
+self.invalids = []
+self.settings = settings
+self._pkg_paths = {}
+self._populating = False
+self._all_directory = os.path.isdir(os.path.join(self.pkgdir, "All"))
+self._pkgindex_version = 0
+self._pkgindex_hashes = ["MD5", "SHA1"]
+self._pkgindex_file = os.path.join(self.pkgdir, "Packages")
+self._pkgindex_keys = self.dbapi._aux_cache_keys.copy()
+self._pkgindex_keys.update(["CPV", "SIZE"])
+self._pkgindex_aux_keys = [
+"BASE_URI",
+"BDEPEND",
+"BINPKG_FORMAT",
+"BUILD_ID",
+"BUILD_TIME",
+"CHOST",
+"DEFINED_PHASES",
+"DEPEND",
+"DESCRIPTION",
+"EAPI",
+"FETCHCOMMAND",
+"IDEPEND",
+"IUSE",
+"KEYWORDS",
+"LICENSE",
+"PDEPEND",
+"PKGINDEX_URI",
+"PROPERTIES",
+"PROVIDES",
+"RDEPEND",
+"repository",
+"REQUIRES",
+"RESTRICT",
+"RESUMECOMMAND",
+"SIZE",
+"SLOT",
+"USE",
+]
+self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
+self._pkgindex_use_evaluated_keys = (
+"BDEPEND",
+"DEPEND",
+"IDEPEND",
+"LICENSE",
+"RDEPEND",
+"PDEPEND",
+"PROPERTIES",
+"RESTRICT",
+)
+self._pkgindex_header = None
+self._pkgindex_header_keys = set(
+[
+"ACCEPT_KEYWORDS",
+"ACCEPT_LICENSE",
+"ACCEPT_PROPERTIES",
+"ACCEPT_RESTRICT",
 "BINPKG_FORMAT",
-"BUILD_ID",
-

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/package/ebuild/, ...

2022-09-09 Thread Michał Górny

commit: fa901a6510c4a5f72dec6aad86db6fe7efd6e7b3
Author: Sheng Yu  protonmail  com>
AuthorDate: Tue Sep  6 18:38:27 2022 +
Commit: Michał Górny  gentoo  org>
CommitDate: Fri Sep  9 10:16:06 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa901a65

Add BUILD_ID to metadata during binary packaging

Also create placeholder for new multi-instance package before actually
packaging to avoid race.

Signed-off-by: Sheng Yu  protonmail.com>
Closes: https://github.com/gentoo/portage/pull/893
Signed-off-by: Michał Górny  gentoo.org>

 bin/gpkg-helper.py |   4 +-
 bin/misc-functions.sh  |   6 +-
 bin/quickpkg   |   2 +-
 lib/_emerge/Binpkg.py  |   2 +-
 lib/_emerge/BinpkgFetcher.py   |  27 ++--
 lib/_emerge/BinpkgPrefetcher.py|   8 +-
 lib/_emerge/EbuildBinpkg.py|  35 +++--
 lib/_emerge/Scheduler.py   |   2 +-
 lib/portage/dbapi/bintree.py   | 145 +++--
 .../package/ebuild/_config/special_env_vars.py |   1 +
 lib/portage/package/ebuild/doebuild.py |   3 +-
 11 files changed, 126 insertions(+), 109 deletions(-)

diff --git a/bin/gpkg-helper.py b/bin/gpkg-helper.py
index d45177f3e..4752c84ee 100755
--- a/bin/gpkg-helper.py
+++ b/bin/gpkg-helper.py
@@ -19,7 +19,7 @@ def command_compose(args):
 sys.stderr.write("4 arguments are required, got %s\n" % len(args))
 return 1
 
-cpv, binpkg_path, metadata_dir, image_dir = args
+basename, binpkg_path, metadata_dir, image_dir = args
 
 if not os.path.isdir(metadata_dir):
 sys.stderr.write(usage)
@@ -31,7 +31,7 @@ def command_compose(args):
 sys.stderr.write("Argument 4 is not a directory: '%s'\n" % image_dir)
 return 1
 
-gpkg_file = portage.gpkg.gpkg(portage.settings, cpv, binpkg_path)
+gpkg_file = portage.gpkg.gpkg(portage.settings, basename, binpkg_path)
 metadata = gpkg_file._generate_metadata_from_dir(metadata_dir)
 gpkg_file.compress(image_dir, metadata)
 return os.EX_OK

diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index faa8184c6..170e60d1c 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -512,6 +512,10 @@ __dyn_package() {
die "PORTAGE_BINPKG_TMPFILE is unset"
mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed"
 
+   if [[ ! -z "${BUILD_ID}" ]]; then
+   echo -n "${BUILD_ID}" > 
"${PORTAGE_BUILDDIR}"/build-info/BUILD_ID
+   fi
+
if [[ "${BINPKG_FORMAT}" == "xpak" ]]; then
local tar_options=""
 
@@ -547,7 +551,7 @@ __dyn_package() {
elif [[ "${BINPKG_FORMAT}" == "gpkg" ]]; then
PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
"${PORTAGE_PYTHON:-/usr/bin/python}" 
"${PORTAGE_BIN_PATH}"/gpkg-helper.py compress \
-   "${CATEGORY}/${PF}" "${PORTAGE_BINPKG_TMPFILE}" 
"${PORTAGE_BUILDDIR}/build-info" "${D}"
+   "${PF}${BUILD_ID:+-${BUILD_ID}}" 
"${PORTAGE_BINPKG_TMPFILE}" "${PORTAGE_BUILDDIR}/build-info" "${D}"
if [[ $? -ne 0 ]]; then
rm -f "${PORTAGE_BINPKG_TMPFILE}"
die "Failed to create binpkg file"

diff --git a/bin/quickpkg b/bin/quickpkg
index 773c1c07e..eef5f912f 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -206,7 +206,7 @@ def quickpkg_atom(options, infos, arg, eout):
 finally:
 if have_lock:
 dblnk.unlockdb()
-pkg_info = bintree.inject(cpv, filename=binpkg_tmpfile)
+pkg_info = bintree.inject(cpv, current_pkg_path=binpkg_tmpfile)
 # The pkg_info value ensures that the following getname call
 # returns the correct path when FEATURES=binpkg-multi-instance
 # is enabled, but fallback to cpv in case the inject call

diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index 15eb56092..949ac8ee7 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -246,7 +246,7 @@ class Binpkg(CompositeTask):
 
 if self._fetched_pkg:
 pkg_path = self._bintree.getname(
-self._bintree.inject(pkg.cpv, filename=self._fetched_pkg),
+self._bintree.inject(pkg.cpv, 
current_pkg_path=self._fetched_pkg),
 allocate_new=False,
 )
 else:

diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index d5275ea11..b7021e276 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -12,6 +12,7 @@ import sys
 import portage
 from portage import os
 from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
+from portage.const import SUPPORTED_XPAK_EXTENSIONS, SUPPORTED_GPKG_EXTENSIONS
 from portage.exception import FileNotFound,

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2022-09-09 Thread Michał Górny

commit: 52411290c67535d94c7b20fa996ae7108014adfb
Author: Sheng Yu  protonmail  com>
AuthorDate: Fri Aug 19 20:24:59 2022 +
Commit: Michał Górny  gentoo  org>
CommitDate: Fri Sep  9 10:15:57 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=52411290

Detect binary package format if not in database

Signed-off-by: Sheng Yu  protonmail.com>
Signed-off-by: Michał Górny  gentoo.org>

 lib/portage/dbapi/bintree.py | 6 ++
 1 file changed, 6 insertions(+)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index b441fff9a..0857ff21a 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1039,6 +1039,12 @@ class binarytree:
 if match:
 binpkg_format = match.get("BINPKG_FORMAT", None)
 
+if not binpkg_format:
+if myfile.endswith(SUPPORTED_XPAK_EXTENSIONS):
+binpkg_format = "xpak"
+elif myfile.endswith(SUPPORTED_GPKG_EXTENSIONS):
+binpkg_format = "gpkg"
+
 if gpkg_only:
 if binpkg_format != "gpkg":
 if not gpkg_only_warned:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/gpkg/, lib/portage/

2022-09-09 Thread Michał Górny

commit: 1d94992a2df2b5cc963c26c7978a899dc642deb1
Author: Sheng Yu  protonmail  com>
AuthorDate: Thu Sep  1 14:44:55 2022 +
Commit: Michał Górny  gentoo  org>
CommitDate: Fri Sep  9 10:15:58 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1d94992a

Move all files into basename/DATA structure

Signed-off-by: Sheng Yu  protonmail.com>
Signed-off-by: Michał Górny  gentoo.org>

 lib/portage/binpkg.py|   8 +-
 lib/portage/dbapi/bintree.py |   2 +-
 lib/portage/gpkg.py  | 201 +--
 lib/portage/tests/gpkg/test_gpkg_checksum.py |  21 +--
 lib/portage/tests/gpkg/test_gpkg_gpg.py  |  28 ++--
 5 files changed, 158 insertions(+), 102 deletions(-)

diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
index ed2fda827..f3f149b14 100644
--- a/lib/portage/binpkg.py
+++ b/lib/portage/binpkg.py
@@ -17,8 +17,8 @@ def get_binpkg_format(binpkg_path):
 
 try:
 with open(binpkg_path, "rb") as binpkg_file:
-header = binpkg_file.read(6)
-if header == b"gpkg-1":
+header = binpkg_file.read(100)
+if b"/gpkg-1\x00" in header:
 file_format = "gpkg"
 else:
 binpkg_file.seek(-16, 2)
@@ -32,7 +32,9 @@ def get_binpkg_format(binpkg_path):
 if file_format is None:
 try:
 with tarfile.open(binpkg_path) as gpkg_tar:
-if "gpkg-1" in gpkg_tar.getnames():
+if "gpkg-1" in [
+f.split("/", maxsplit=1)[-1] for f in 
gpkg_tar.getnames()
+]:
 file_format = "gpkg"
 except tarfile.TarError:
 pass

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 0857ff21a..814e6627c 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -744,7 +744,7 @@ class binarytree:
 mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
 elif binpkg_format == "gpkg":
 mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, 
update_path)
-mybinpkg.update_metadata(mydata, newcpv=mynewcpv)
+mybinpkg.update_metadata(mydata, new_basename=mynewcpv)
 else:
 raise InvalidBinaryPackageFormat(binpkg_format)
 self.inject(mynewcpv, filename=update_path)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 644ff412b..5f8e19341 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -764,10 +764,10 @@ class gpkg:
 https://www.gentoo.org/glep/glep-0078.html
 """
 
-def __init__(self, settings, base_name=None, gpkg_file=None):
+def __init__(self, settings, basename=None, gpkg_file=None):
 """
 gpkg class handle all gpkg operations for one package.
-base_name is the package basename.
+basename is the package basename.
 gpkg_file should be exists file path for read or will create.
 """
 self.settings = settings
@@ -778,10 +778,16 @@ class gpkg:
 self.gpkg_file = _unicode_decode(
 gpkg_file, encoding=_encodings["fs"], errors="strict"
 )
-self.base_name = base_name
+
+if basename is None:
+self.basename = None
+else:
+self.basename = basename.split("/", maxsplit=1)[-1]
+
 self.checksums = []
 self.manifest_old = []
-signature_exist = None
+self.signature_exist = None
+self.prefix = None
 
 # Compression is the compression algorithm, if set to None will
 # not use compression.
@@ -892,7 +898,9 @@ class gpkg:
 
 # Check gpkg and metadata
 with tarfile.open(mode="r", fileobj=container_file) as container:
-if self.gpkg_version not in container.getnames():
+if self.gpkg_version not in (
+os.path.basename(f) for f in container.getnames()
+):
 raise InvalidBinaryPackageFormat("Invalid gpkg file.")
 
 metadata_tarinfo, metadata_comp = self._get_inner_tarinfo(
@@ -985,7 +993,7 @@ class gpkg:
 )
 
 # Long CPV
-if len(self.base_name) >= 154:
+if len(self.basename) >= 154:
 container_tar_format = tarfile.GNU_FORMAT
 
 # gpkg container
@@ -994,9 +1002,14 @@ class gpkg:
 )
 
 # gpkg version
-gpkg_version_file = tarfile.TarInfo(self.gpkg_version)
+gpkg_version_file = tarfile.TarInfo(
+os.path.join(self.basename, self.gpkg_version)
+)
 gpkg_version_file.mtime = datetime.utcnow().timestamp()
 container.addfile(gpkg_version_file)
+checksum_info = checksum_helper(self.settings)
+checksum_info.finish()
+self._record_checksum(checksum_info, gpkg_version_file)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2022-08-18 Thread Mike Gilbert

commit: 57ce385e32e79b9d332fe1fdb3be50bdb07e7838
Author: Mike Gilbert  gentoo  org>
AuthorDate: Thu Aug 18 17:59:03 2022 +
Commit: Mike Gilbert  gentoo  org>
CommitDate: Thu Aug 18 18:11:25 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=57ce385e

dbapi: avoid iterating porttrees twice in _set_porttrees()

If porttrees is a generator object, the second pass will fail.

Bug: https://bugs.gentoo.org/865635
Fixes: 9e24d0143450628f334cdb62e579efafd1bfd2ba
Signed-off-by: Mike Gilbert  gentoo.org>

 lib/portage/dbapi/porttree.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index cd919ba31..126e7161b 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -367,14 +367,14 @@ class portdbapi(dbapi):
 repo priority
 @type porttrees: list
 """
+self._porttrees = tuple(porttrees)
 self._porttrees_repos = portage.OrderedDict(
 (repo.name, repo)
 for repo in (
 self.repositories.get_repo_for_location(location)
-for location in porttrees
+for location in self._porttrees
 )
 )
-self._porttrees = tuple(porttrees)
 
 def _get_porttrees(self):
 return self._porttrees

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2022-04-20 Thread Zac Medico

commit: e893f4fc12eb618318b1945ce7a05a94fb1ea1b4
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 17 19:12:40 2022 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Apr 18 02:18:31 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e893f4fc

MergeProcess: propagate mtimedb["ldpath"] to parent process (bug 836375)

Use an instance of multiprocessing.Pipe to propagate mtimedb["ldpath"]
from the MergeProcess child process to the parent process. This fixes
env_update calls to avoid unnecessary regeneration of ld.so.cache in
cases where mtimedb["ldpath"] has not changed since the last call to
env_update.

Bug: https://bugs.gentoo.org/836375
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/_MergeProcess.py | 22 ++
 lib/portage/dbapi/vartree.py   | 10 ++
 2 files changed, 32 insertions(+)

diff --git a/lib/portage/dbapi/_MergeProcess.py 
b/lib/portage/dbapi/_MergeProcess.py
index db3f3b105..197c48a7e 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import io
+import multiprocessing
 import platform
 
 import fcntl
@@ -38,6 +39,7 @@ class MergeProcess(ForkProcess):
 "_dblink",
 "_elog_keys",
 "_locked_vdb",
+"_mtime_reader",
 )
 
 def _start(self):
@@ -113,6 +115,19 @@ class MergeProcess(ForkProcess):
 self._elog_reader_fd = None
 return False
 
+def _mtime_handler(self):
+if self._mtime_reader is not None:
+try:
+mtimes = self._mtime_reader.recv()
+except EOFError:
+self.scheduler.remove_reader(self._mtime_reader.fileno())
+self._mtime_reader.close()
+self._mtime_reader = None
+else:
+if self.prev_mtimes is not None:
+self.prev_mtimes.clear()
+self.prev_mtimes.update(mtimes)
+
 def _spawn(self, args, fd_pipes, **kwargs):
 """
 Extend the superclass _spawn method to perform some pre-fork and
@@ -127,6 +142,11 @@ class MergeProcess(ForkProcess):
 fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
 )
 
+mtime_reader, mtime_writer = multiprocessing.Pipe(duplex=False)
+fd_pipes[mtime_writer.fileno()] = mtime_writer.fileno()
+self.scheduler.add_reader(mtime_reader.fileno(), self._mtime_handler)
+self._mtime_reader = mtime_reader
+
 blockers = None
 if self.blockers is not None:
 # Query blockers in the main process, since closing
@@ -142,6 +162,7 @@ class MergeProcess(ForkProcess):
 vartree=self.vartree,
 blockers=blockers,
 pipe=elog_writer_fd,
+mtime_pipe=mtime_writer,
 )
 fd_pipes[elog_writer_fd] = elog_writer_fd
 self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler)
@@ -160,6 +181,7 @@ class MergeProcess(ForkProcess):
 self._elog_reader_fd = elog_reader_fd
 pids = super(MergeProcess, self)._spawn(args, fd_pipes, **kwargs)
 os.close(elog_writer_fd)
+mtime_writer.close()
 self._buf = ""
 self._elog_keys = set()
 # Discard messages which will be collected by the subprocess,

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 602913862..a95d60691 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1806,6 +1806,7 @@ class dblink:
 blockers=None,
 scheduler=None,
 pipe=None,
+mtime_pipe=None,
 ):
 """
 Creates a DBlink object for a given CPV.
@@ -1862,6 +1863,7 @@ class dblink:
 self._device_path_map = {}
 self._hardlink_merge_map = {}
 self._hash_key = (self._eroot, self.mycpv)
+self._mtime_pipe = mtime_pipe
 self._protect_obj = None
 self._pipe = pipe
 self._postinst_failure = False
@@ -2618,6 +2620,7 @@ class dblink:
 writemsg_level=self._display_merge,
 vardbapi=self.vartree.dbapi,
 )
+self._send_mtimes(ldpath_mtimes)
 
 unmerge_with_replacement = preserve_paths is not None
 if not unmerge_with_replacement:
@@ -4243,6 +4246,12 @@ class dblink:
 def _emerge_log(self, msg):
 emergelog(False, msg)
 
+def _send_mtimes(self, mtimes):
+if self._mtime_pipe is None:
+return
+
+self._mtime_pipe.send(mtimes)
+
 def treewalk(
 self,
 srcroot,
@@ -5274,6 +5283,7 @@ class dblink:
 writemsg_level=self._display_merge,
 vardbapi=self.vartree.dbapi,
 )
+self._send_mtimes(prev_mtimes)
 
 # For gcc upgrades, preserved libs have to be removed after the
 # the library path has been

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2022-04-13 Thread Sam James

commit: 05c0b3d03393fe376f3ae4b49bce403d496e3e68
Author: Mike Frysinger  chromium  org>
AuthorDate: Tue Mar 29 07:12:25 2022 +
Commit: Sam James  gentoo  org>
CommitDate: Wed Apr 13 15:34:30 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=05c0b3d0

vartree: avoid lock contention when there are no blockers

No sense in grabbing the vdb lock if we aren't going to do any work.
This avoids contention on the global lock with parallel packages.

[sam: cherry-picked from chromiumos' third_party/portage_tool repo]
(cherry picked from commit ea5f6f8c0a5e05d7630f9070992a89fa6907cc14)
Signed-off-by: Sam James  gentoo.org>
Closes: https://github.com/gentoo/portage/pull/813
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/vartree.py | 19 +++
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index e7252790d..602913862 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -5148,14 +5148,17 @@ class dblink:
 self._clear_contents_cache()
 contents = self.getcontents()
 destroot_len = len(destroot) - 1
-self.lockdb()
-try:
-for blocker in blockers:
-self.vartree.dbapi.removeFromContents(
-blocker, iter(contents), relative_paths=False
-)
-finally:
-self.unlockdb()
+
+# Avoid lock contention if we aren't going to do any work.
+if blockers:
+self.lockdb()
+try:
+for blocker in blockers:
+self.vartree.dbapi.removeFromContents(
+blocker, iter(contents), relative_paths=False
+)
+finally:
+self.unlockdb()
 
 plib_registry = self.vartree.dbapi._plib_registry
 if plib_registry:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2022-04-13 Thread Sam James

commit: 964c0e16172b76a8ebdd737cd8919870ae2b5f96
Author: Mike Frysinger  chromium  org>
AuthorDate: Tue Mar 29 13:56:55 2022 +
Commit: Sam James  gentoo  org>
CommitDate: Wed Apr 13 15:34:30 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=964c0e16

vartree: skip env-update if no updates were merged

This speeds up virtual/ installs by not constantly re-running env-update.

[sam: cherry-picked from chromiumos' third_party/portage_tool repo]
(cherry picked from commit 87ac3566ebb7155a57876d345849bd0fd6878c0e)

Bug: https://bugs.gentoo.org/836375
Signed-off-by: Sam James  gentoo.org>

 lib/portage/dbapi/vartree.py | 38 +-
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 072bc1506..e7252790d 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -2608,14 +2608,16 @@ class dblink:
 else:
 self.settings.pop("PORTAGE_LOG_FILE", None)
 
-env_update(
-target_root=self.settings["ROOT"],
-prev_mtimes=ldpath_mtimes,
-contents=contents,
-env=self.settings,
-writemsg_level=self._display_merge,
-vardbapi=self.vartree.dbapi,
-)
+# If we didn't unmerge anything, don't bother updating env.
+if contents:
+env_update(
+target_root=self.settings["ROOT"],
+prev_mtimes=ldpath_mtimes,
+contents=contents,
+env=self.settings,
+writemsg_level=self._display_merge,
+vardbapi=self.vartree.dbapi,
+)
 
 unmerge_with_replacement = preserve_paths is not None
 if not unmerge_with_replacement:
@@ -5258,15 +5260,17 @@ class dblink:
 ],
 )
 
-# update environment settings, library paths. DO NOT change symlinks.
-env_update(
-target_root=self.settings["ROOT"],
-prev_mtimes=prev_mtimes,
-contents=contents,
-env=self.settings,
-writemsg_level=self._display_merge,
-vardbapi=self.vartree.dbapi,
-)
+# Update environment settings, library paths. DO NOT change symlinks.
+# Only do this if we actually installed something.
+if contents:
+env_update(
+target_root=self.settings["ROOT"],
+prev_mtimes=prev_mtimes,
+contents=contents,
+env=self.settings,
+writemsg_level=self._display_merge,
+vardbapi=self.vartree.dbapi,
+)
 
 # For gcc upgrades, preserved libs have to be removed after the
 # the library path has been updated.

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/_emerge/, cnf/

2022-04-12 Thread Sam James

commit: eecd073d274c6d669f324cbb31151789d32f87fb
Author: Sam James  gentoo  org>
AuthorDate: Sat Mar  5 05:57:25 2022 +
Commit: Sam James  gentoo  org>
CommitDate: Wed Apr 13 03:05:45 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=eecd073d

autoclean: only enable for global updates and depcleans

Closes: https://bugs.gentoo.org/792195
Signed-off-by: Sam James  gentoo.org>
Closes: https://github.com/gentoo/portage/pull/795
Signed-off-by: Sam James  gentoo.org>

 cnf/make.globals |  2 +-
 lib/_emerge/actions.py   | 22 --
 lib/portage/dbapi/vartree.py | 24 +---
 3 files changed, 22 insertions(+), 26 deletions(-)

diff --git a/cnf/make.globals b/cnf/make.globals
index 69b365f71..f42cffe8b 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -101,7 +101,7 @@ EMERGE_WARNING_DELAY="10"
 
 # Automatically clean installed packages after they are updated.
 # This option will be removed and forced to yes.
-AUTOCLEAN="yes"
+AUTOCLEAN="no"
 
 PORTAGE_BZIP2_COMMAND="bzip2"
 

diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index f82069f9a..e697459c8 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -672,12 +672,6 @@ def action_build(
 ldpath_mtimes,
 autoclean=1,
 )
-else:
-portage.writemsg_stdout(
-colorize("WARN", "WARNING:")
-+ " AUTOCLEAN is disabled.  This can cause serious"
-+ " problems due to overlapping packages.\n"
-)
 
 return retval
 
@@ -776,6 +770,15 @@ def action_depclean(
 # relatively safe to ignore missing deps when only asked to remove
 # specific packages.
 
+# Force autoclean for depcleans (but not purges), as it was changed
+# to default off to not run it on every unmerge.
+# bug #792195
+if action == "depclean":
+settings.unlock()
+settings["AUTOCLEAN"] = "yes"
+settings.backup_changes("AUTOCLEAN")
+settings.lock()
+
 msg = []
 if (
 "preserve-libs" not in settings.features
@@ -3386,6 +3389,13 @@ def run_action(emerge_config):
 # Reload the whole config from scratch.
 load_emerge_config(emerge_config=emerge_config)
 
+# Let's autoclean if we applied updates, rather than always doing it
+# bug #792195
+emerge_config.target_config.settings.unlock()
+emerge_config.target_config.settings["AUTOCLEAN"] = "yes"
+emerge_config.target_config.settings.backup_changes("AUTOCLEAN")
+emerge_config.target_config.settings.lock()
+
 xterm_titles = "notitles" not in 
emerge_config.target_config.settings.features
 if xterm_titles:
 xtermTitle("emerge")

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 863efe9cc..190bf8c92 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -5086,21 +5086,18 @@ class dblink:
 
 emerge_log = self._emerge_log
 
-# If we have any preserved libraries then autoclean
-# is forced so that preserve-libs logic doesn't have
+# We always autoclean now for the current package-case for simplicity.
+# If it were conditional, we'd always need to do it when any 
preserved-libs,
+# so that preserve-libs logic doesn't have
 # to account for the additional complexity of the
 # AUTOCLEAN=no mode.
-autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes" or 
preserve_paths
-
-if autoclean:
-emerge_log(_(" >>> AUTOCLEAN: %s") % (slot_atom,))
+emerge_log(_(" >>> AUTOCLEAN: %s") % (slot_atom,))
 
 others_in_slot.append(self)  # self has just been merged
 for dblnk in list(others_in_slot):
 if dblnk is self:
 continue
-if not (autoclean or dblnk.mycpv == self.mycpv or reinstall_self):
-continue
+
 showMessage(_(">>> Safely unmerging already-installed 
instance...\n"))
 emerge_log(_(" === Unmerging... (%s)") % (dblnk.mycpv,))
 others_in_slot.remove(dblnk)  # dblnk will unmerge itself now
@@ -5130,17 +5127,6 @@ class dblink:
 self.unlockdb()
 showMessage(_(">>> Original instance of package unmerged 
safely.\n"))
 
-if len(others_in_slot) > 1:
-showMessage(
-colorize("WARN", _("WARNING:"))
-+ _(
-" AUTOCLEAN is disabled.  This can cause serious"
-" problems due to overlapping packages.\n"
-),
-level=logging.WARN,
-noiselevel=-1,
-)
-
 # We hold both directory locks.
 self.dbdir = self.dbpkgdir
 self.lockdb()

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-11-26 Thread Mike Gilbert

commit: 0009404ee5f293fe7076d59a52c46827c3b77738
Author: Mike Gilbert  gentoo  org>
AuthorDate: Fri Nov 26 17:43:45 2021 +
Commit: Mike Gilbert  gentoo  org>
CommitDate: Fri Nov 26 17:43:45 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0009404e

dbapi: fix logic in bintree.binarytree._parse_build_id()

Resolves an error when improperly named xpak files exist in PKGDIR.

Bug: https://bugs.gentoo.org/818886
Signed-off-by: Mike Gilbert  gentoo.org>

 lib/portage/dbapi/bintree.py | 9 -
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 777fc4918..9dbf9ee8b 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1877,11 +1877,10 @@ class binarytree:
 suffixlen = len(".xpak")
 hyphen = filename.rfind("-", 0, -(suffixlen + 1))
 if hyphen != -1:
-build_id = filename[hyphen + 1 : -suffixlen]
-try:
-build_id = int(build_id)
-except ValueError:
-pass
+try:
+build_id = int(filename[hyphen + 1 : -suffixlen])
+except ValueError:
+pass
 return build_id
 
 def isremote(self, pkgname):

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-09-20 Thread Zac Medico

commit: 95935004862d1363a0e3d5b6ee33d4ed65566a2d
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Sep 21 05:13:12 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Sep 21 05:33:51 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=95935004

portdbapi: convert compat coroutine to async

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/porttree.py | 10 --
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 72b875879..93f3fee2f 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -38,7 +38,6 @@ from portage import os
 from portage import _encodings
 from portage import _unicode_encode
 from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import coroutine, coroutine_return
 from portage.util.futures.iter_completed import iter_gather
 from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
 
@@ -1244,8 +1243,7 @@ class portdbapi(dbapi):
 loop = self._event_loop
 return loop.run_until_complete(self.async_xmatch(level, origdep, 
loop=loop))
 
-@coroutine
-def async_xmatch(self, level, origdep, loop=None):
+async def async_xmatch(self, level, origdep, loop=None):
 """
 Asynchronous form of xmatch.
 
@@ -1269,7 +1267,7 @@ class portdbapi(dbapi):
 if self.frozen:
 cache_key = (mydep, mydep.unevaluated_atom)
 try:
-coroutine_return(self.xcache[level][cache_key][:])
+return self.xcache[level][cache_key][:]
 except KeyError:
 pass
 
@@ -1336,7 +1334,7 @@ class portdbapi(dbapi):
 zip(
 aux_keys,
 (
-yield self.async_aux_get(
+await self.async_aux_get(
 cpv, aux_keys, myrepo=cpv.repo, loop=loop
 )
 ),
@@ -1384,7 +1382,7 @@ class portdbapi(dbapi):
 if not isinstance(myval, _pkg_str):
 myval = myval[:]
 
-coroutine_return(myval)
+return myval
 
 def match(self, mydep, use_cache=1):
 return self.xmatch("match-visible", mydep)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-09-20 Thread Zac Medico

commit: bcda30d0a6fa4962cc7597dbed9b648cf6400ab5
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Sep 21 05:21:55 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Sep 21 05:33:51 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bcda30d0

vardbapi: convert compat coroutine to async

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py | 15 ++-
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 7f3b5d773..8ffb23b1c 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -78,7 +78,6 @@ from portage import _os_merge
 from portage import _selinux_merge
 from portage import _unicode_decode
 from portage import _unicode_encode
-from portage.util.futures.compat_coroutine import coroutine
 from portage.util.futures.executor.fork import ForkExecutor
 from ._VdbMetadataDelta import VdbMetadataDelta
 
@@ -1006,8 +1005,7 @@ class vardbapi(dbapi):
 pass
 self._bump_mtime(cpv)
 
-@coroutine
-def unpack_metadata(self, pkg, dest_dir, loop=None):
+async def unpack_metadata(self, pkg, dest_dir, loop=None):
 """
 Unpack package metadata to a directory. This method is a coroutine.
 
@@ -1029,10 +1027,9 @@ class vardbapi(dbapi):
 shutil.copy(os.path.join(parent, key), 
os.path.join(dest_dir, key))
 break
 
-yield loop.run_in_executor(ForkExecutor(loop=loop), async_copy)
+await loop.run_in_executor(ForkExecutor(loop=loop), async_copy)
 
-@coroutine
-def unpack_contents(
+async def unpack_contents(
 self,
 pkg,
 dest_dir,
@@ -1097,10 +1094,10 @@ class vardbapi(dbapi):
 
 tar_cmd = ("tar", "-x", "--xattrs", "--xattrs-include=*", "-C", 
dest_dir)
 pr, pw = os.pipe()
-proc = yield asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)
+proc = await asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)
 os.close(pr)
 with os.fdopen(pw, "wb", 0) as pw_file:
-excluded_config_files = yield loop.run_in_executor(
+excluded_config_files = await loop.run_in_executor(
 ForkExecutor(loop=loop),
 functools.partial(
 self._dblink(cpv).quickpkg,
@@ -1109,7 +1106,7 @@ class vardbapi(dbapi):
 include_unmodified_config=opts.include_unmodified_config 
== "y",
 ),
 )
-yield proc.wait()
+await proc.wait()
 if proc.returncode != os.EX_OK:
 raise PortageException("command failed: {}".format(tar_cmd))

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-09-20 Thread Zac Medico

commit: 7957a7f4c2ef4656f6f9a29d2c9bab6e44daae1f
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Sep 21 05:08:21 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Sep 21 05:33:51 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7957a7f4

bindbapi: convert compat coroutine to async

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 15 ++-
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 024a9b5f6..777fc4918 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -39,7 +39,6 @@ from portage.localization import _
 from portage.package.ebuild.profile_iuse import iter_iuse_vars
 from portage.util.file_copy import copyfile
 from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import coroutine
 from portage.util.futures.executor.fork import ForkExecutor
 from portage import _movefile
 from portage import os
@@ -253,8 +252,7 @@ class bindbapi(fakedbapi):
 # inject will clear stale caches via cpv_inject.
 self.bintree.inject(cpv)
 
-@coroutine
-def unpack_metadata(self, pkg, dest_dir, loop=None):
+async def unpack_metadata(self, pkg, dest_dir, loop=None):
 """
 Unpack package metadata to a directory. This method is a coroutine.
 
@@ -271,17 +269,16 @@ class bindbapi(fakedbapi):
 key = self._instance_key(cpv)
 add_pkg = self.bintree._additional_pkgs.get(key)
 if add_pkg is not None:
-yield add_pkg._db.unpack_metadata(pkg, dest_dir, loop=loop)
+await add_pkg._db.unpack_metadata(pkg, dest_dir, loop=loop)
 else:
 tbz2_file = self.bintree.getname(cpv)
-yield loop.run_in_executor(
+await loop.run_in_executor(
 ForkExecutor(loop=loop),
 portage.xpak.tbz2(tbz2_file).unpackinfo,
 dest_dir,
 )
 
-@coroutine
-def unpack_contents(self, pkg, dest_dir, loop=None):
+async def unpack_contents(self, pkg, dest_dir, loop=None):
 """
 Unpack package contents to a directory. This method is a coroutine.
 
@@ -313,7 +310,7 @@ class bindbapi(fakedbapi):
 )
 
 extractor.start()
-yield extractor.async_wait()
+await extractor.async_wait()
 if extractor.returncode != os.EX_OK:
 raise PortageException("Error Extracting 
'{}'".format(pkg_path))
 
@@ -322,7 +319,7 @@ class bindbapi(fakedbapi):
 add_pkg = self.bintree._additional_pkgs.get(instance_key)
 if add_pkg is None:
 raise portage.exception.PackageNotFound(cpv)
-yield add_pkg._db.unpack_contents(pkg, dest_dir, loop=loop)
+await add_pkg._db.unpack_contents(pkg, dest_dir, loop=loop)
 
 def cp_list(self, *pargs, **kwargs):
 if not self.bintree.populated:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-06-05 Thread Zac Medico

commit: 2fe8238b540240a62de3513839974e033f0bacd2
Author: Daniel M. Weeks  danweeks  net>
AuthorDate: Mon May 31 16:10:21 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Jun  5 18:03:45 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2fe8238b

Support GLEP 75 in PORTAGE_RO_DISTDIRS size check

Closes: https://github.com/gentoo/portage/pull/724
Signed-off-by: Daniel M. Weeks  danweeks.net>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/porttree.py | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 99c36450c..0f50f1338 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -12,7 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.dbapi.dep_expand:dep_expand',
'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot',
'portage.package.ebuild.doebuild:doebuild',
-   'portage.package.ebuild.fetch:_download_suffix',
+   'portage.package.ebuild.fetch:get_mirror_url,_download_suffix',
'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
'portage.util.listdir:listdir',

'portage.versions:best,catsplit,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str',
@@ -859,7 +859,11 @@ class portdbapi(dbapi):
if ro_distdirs is not None:
for x in shlex_split(ro_distdirs):
try:
-   mystat = 
os.stat(os.path.join(x, myfile))
+   mystat = os.stat(
+   
portage.package.ebuild.fetch.get_mirror_url(
+   x, 
myfile, self.settings
+   )
+   )
except OSError:
pass
else:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/_emerge/

2021-03-11 Thread Zac Medico

commit: 8879cef9006d2277453aaee407c234a2d1bc47ba
Author: Florian Schmaus  geekplace  eu>
AuthorDate: Tue Mar  9 07:25:59 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Mar 11 12:04:41 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8879cef9

Use atomic_ofstream as Context Manager i.e., with-statement contexts

With [1: e93e6d65fa1c] atomic_ofstream became a Context Manager. This
commit transforms three further call sites of atomic_ofstream() to use
with-statement contexts for easier readability and increased
robustness against resource leaks.

1: e93e6d65fa1ca75f676a227f7918f8b6d747425c
   Make atomic_ofstream a Context Manager

Signed-off-by: Florian Schmaus  geekplace.eu>
Signed-off-by: Zac Medico  gentoo.org>

 lib/_emerge/BlockerCache.py|  6 +++---
 lib/portage/dbapi/_VdbMetadataDelta.py | 11 +--
 lib/portage/dbapi/vartree.py   |  6 +++---
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/lib/_emerge/BlockerCache.py b/lib/_emerge/BlockerCache.py
index 8154d9ade..035f2212d 100644
--- a/lib/_emerge/BlockerCache.py
+++ b/lib/_emerge/BlockerCache.py
@@ -133,9 +133,9 @@ class BlockerCache(portage.cache.mappings.MutableMapping):
if len(self._modified) >= self._cache_threshold and \
secpass >= 2:
try:
-   f = 
portage.util.atomic_ofstream(self._cache_filename, mode='wb')
-   pickle.dump(self._cache_data, f, protocol=2)
-   f.close()
+   with 
portage.util.atomic_ofstream(self._cache_filename, mode='wb') as f:
+   pickle.dump(self._cache_data, f, 
protocol=2)
+
portage.util.apply_secpass_permissions(
self._cache_filename, 
gid=portage.portage_gid, mode=0o644)
except (IOError, OSError):

diff --git a/lib/portage/dbapi/_VdbMetadataDelta.py 
b/lib/portage/dbapi/_VdbMetadataDelta.py
index ffdc0b361..568e1964a 100644
--- a/lib/portage/dbapi/_VdbMetadataDelta.py
+++ b/lib/portage/dbapi/_VdbMetadataDelta.py
@@ -18,13 +18,12 @@ class VdbMetadataDelta:
self._vardb = vardb
 
def initialize(self, timestamp):
-   f = atomic_ofstream(self._vardb._cache_delta_filename, 'w',
-   encoding=_encodings['repo.content'], errors='strict')
-   json.dump({
-   "version": self._format_version,
-   "timestamp": timestamp
+   with atomic_ofstream(self._vardb._cache_delta_filename, 'w',
+   encoding=_encodings['repo.content'], errors='strict') 
as f:
+   json.dump({
+   "version": self._format_version,
+   "timestamp": timestamp
}, f, ensure_ascii=False)
-   f.close()
 
def load(self):
 

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 826083eae..5ae035baf 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -614,9 +614,9 @@ class vardbapi(dbapi):
timestamp = time.time()
self._aux_cache["timestamp"] = timestamp
 
-   f = atomic_ofstream(self._aux_cache_filename, 'wb')
-   pickle.dump(self._aux_cache, f, protocol=2)
-   f.close()
+   with atomic_ofstream(self._aux_cache_filename, 'wb') as 
f:
+   pickle.dump(self._aux_cache, f, protocol=2)
+
apply_secpass_permissions(
self._aux_cache_filename, mode=0o644)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-03-07 Thread Zac Medico

commit: 6eafeb89fb4d17326d6ffa1d6ca83e5407890d42
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Mar  7 09:03:33 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Mar  7 09:30:27 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6eafeb89

dblink: add _format_contents_line method

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py | 42 +-
 1 file changed, 37 insertions(+), 5 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 2c75be4a1..826083eae 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -5045,7 +5045,14 @@ class dblink:
% (relative_path, 
myabsto)])
 
showMessage("%s %s -> %s\n" % (zing, 
mydest, myto))
-   outfile.write("sym "+myrealdest+" -> 
"+myto+" "+str(mymtime // 10)+"\n")
+   outfile.write(
+   self._format_contents_line(
+   node_type="sym",
+   abs_path=myrealdest,
+   symlink_target=myto,
+   mtime_ns=mymtime,
+   )
+   )
else:
showMessage(_("!!! Failed to move 
file.\n"),
level=logging.ERROR, 
noiselevel=-1)
@@ -5146,7 +5153,9 @@ class dblink:
except OSError:
pass
 
-   outfile.write("dir "+myrealdest+"\n")
+   outfile.write(
+   
self._format_contents_line(node_type="dir", abs_path=myrealdest)
+   )
# recurse and merge this directory
mergelist.extend(join(relative_path, child) for 
child in
os.listdir(join(srcroot, 
relative_path)))
@@ -5194,7 +5203,14 @@ class dblink:
pass
 
if mymtime != None:
-   outfile.write("obj "+myrealdest+" 
"+mymd5+" "+str(mymtime // 10)+"\n")
+   outfile.write(
+   self._format_contents_line(
+   node_type="obj",
+   abs_path=myrealdest,
+   md5_digest=mymd5,
+   mtime_ns=mymtime,
+   )
+   )
showMessage("%s %s\n" % (zing,mydest))
else:
# we are merging a fifo or device node
@@ -5214,9 +5230,13 @@ class dblink:
else:
return 1
if stat.S_ISFIFO(mymode):
-   outfile.write("fif %s\n" % myrealdest)
+   outfile.write(
+   
self._format_contents_line(node_type="fif", abs_path=myrealdest)
+   )
else:
-   outfile.write("dev %s\n" % myrealdest)
+   outfile.write(
+   
self._format_contents_line(node_type="dev", abs_path=myrealdest)
+   )
showMessage(zing + " " + mydest + "\n")
 
def _protect(self, cfgfiledict, protect_if_modified, src_md5,
@@ -5278,6 +5298,18 @@ class dblink:
 
return dest, protected, move_me
 
+   def _format_contents_line(
+   self, node_type, abs_path, md5_digest=None, 
symlink_target=None, mtime_ns=None
+   ):
+   fields = [node_type, abs_path]
+   if md5_digest is not None:
+   fields.append(md5_digest)
+   elif symlink_target is not None:
+   fields.append("-> {}".format(symlink_target))
+   if mtime_ns is not None:
+   fields.append(str(mtime_ns // 10))
+   return "{}\n".format(" ".join(fields))
+
def _merged_path(self, path, lstatobj,

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-02-23 Thread Zac Medico

commit: a2c59fa326ecacf9834df574a81f4a5e5b7d6b93
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Feb 23 21:29:57 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Feb 23 21:30:19 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a2c59fa3

binarytree.inject: create PKGDIR

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index ab09b42bc..429c8e64a 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1255,6 +1255,7 @@ class binarytree:
# process) and then updated it, all while holding a lock.
pkgindex_lock = None
try:
+   os.makedirs(self.pkgdir, exist_ok=True)
pkgindex_lock = lockfile(self._pkgindex_file,
wantnewlockfile=1)
if filename is not None:

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-01-18 Thread Zac Medico

commit: 7007dbcd1013829466498d7e0708c8b84fdd68bf
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Jan 18 08:45:34 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Jan 18 08:46:44 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7007dbcd

bindbapi.aux_update: fix spurious package file renames

Omit the filename argument for the binarytree.inject method
in aux_update, since an in-place update is desired, and the
filename argument causes the file to be renamed when
binpkg-multi-instance is enabled.

Fixes: 328dd4712f88 ("binpkg-multi-instance 3 of 7")
Bug: https://bugs.gentoo.org/765847
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index e5ba5893e..180e48c3b 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -214,7 +214,7 @@ class bindbapi(fakedbapi):
del mydata[k]
mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
# inject will clear stale caches via cpv_inject.
-   self.bintree.inject(cpv, filename=tbz2path)
+   self.bintree.inject(cpv)
 
 
@coroutine

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-01-17 Thread Zac Medico

commit: 3d39e6891a3ee002fb1aa039c5660e6015c555d2
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jan 17 13:24:21 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jan 17 13:27:25 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d39e689

bindbapi.move_ent: use cpv_exists instead of getname

The getname method is complicated by binpkg-multi-instance
and soon BINPKG_FORMAT, so it's much simpler to use
cpv_exists.

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 528a68979..e5ba5893e 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -483,7 +483,7 @@ class binarytree:
myoldpkg = catsplit(mycpv)[1]
mynewpkg = catsplit(mynewcpv)[1]
 
-   if (mynewpkg != myoldpkg) and 
os.path.exists(self.getname(mynewcpv)):
+   if (mynewpkg != myoldpkg) and 
self.dbapi.cpv_exists(mynewcpv):
writemsg(_("!!! Cannot update binary: 
Destination exists.\n"),
noiselevel=-1)
writemsg("!!! "+mycpv+" -> "+mynewcpv+"\n", 
noiselevel=-1)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-01-17 Thread Zac Medico

commit: 375eb5c02df19833ddeb2c63438aac9faf3d5eae
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jan 17 08:35:41 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jan 17 08:46:23 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=375eb5c0

vardbapi.move_ent: remove redundant self._pkg_str calls

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py | 8 ++--
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index f3d74cf82..2c75be4a1 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2020 Gentoo Authors
+# Copyright 1998-2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = [
@@ -372,11 +372,7 @@ class vardbapi(dbapi):
if not origmatches:
return moves
for mycpv in origmatches:
-   try:
-   mycpv = self._pkg_str(mycpv, None)
-   except (KeyError, InvalidData):
-   continue
-   mycpv_cp = cpv_getkey(mycpv)
+   mycpv_cp = mycpv.cp
if mycpv_cp != origcp:
# Ignore PROVIDE virtual match.
continue

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2021-01-17 Thread Zac Medico

commit: bdc75b3aa217649f6d835b04eb3cac879459c0a0
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jan 17 08:47:13 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jan 17 08:47:26 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bdc75b3a

bindbapi.move_ent: remove redundant self._pkg_str calls

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 10 +++---
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 7e24589e5..528a68979 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2020 Gentoo Authors
+# Copyright 1998-2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["bindbapi", "binarytree"]
@@ -27,7 +27,7 @@ from portage.cache.mappings import slot_dict_class
 from portage.const import BINREPOS_CONF_FILE, CACHE_PATH, 
SUPPORTED_XPAK_EXTENSIONS
 from portage.dbapi.virtual import fakedbapi
 from portage.dep import Atom, use_reduce, paren_enclose
-from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
+from portage.exception import AlarmSignal, InvalidPackageName, \
ParseError, PortageException
 from portage.localization import _
 from portage.package.ebuild.profile_iuse import iter_iuse_vars
@@ -466,11 +466,7 @@ class binarytree:
if not origmatches:
return moves
for mycpv in origmatches:
-   try:
-   mycpv = self.dbapi._pkg_str(mycpv, None)
-   except (KeyError, InvalidData):
-   continue
-   mycpv_cp = portage.cpv_getkey(mycpv)
+   mycpv_cp = mycpv.cp
if mycpv_cp != origcp:
# Ignore PROVIDE virtual match.
continue

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/package/ebuild/

2020-12-06 Thread Zac Medico

commit: 87ca3d4b09324a2359ea10aa69c1f2875c531486
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Dec  7 02:54:26 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Dec  7 02:54:52 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=87ca3d4b

Remove unused EventLoop lazy imports

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py   | 1 -
 lib/portage/package/ebuild/doebuild.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 1547d2f6d..f3d74cf82 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -41,7 +41,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap',
'portage.util._dyn_libs.NeededEntry:NeededEntry',
'portage.util._async.SchedulerInterface:SchedulerInterface',
-   'portage.util._eventloop.EventLoop:EventLoop',
'portage.util._eventloop.global_event_loop:global_event_loop',
'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,' + \
'_get_slot_re,_pkgsplit@pkgsplit,_pkg_str,_unknown_repo',

diff --git a/lib/portage/package/ebuild/doebuild.py 
b/lib/portage/package/ebuild/doebuild.py
index 3b1991b28..f6cee4518 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -39,7 +39,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util._dyn_libs.NeededEntry:NeededEntry',
'portage.util._dyn_libs.soname_deps:SonameDepsProcessor',
'portage.util._async.SchedulerInterface:SchedulerInterface',
-   'portage.util._eventloop.EventLoop:EventLoop',
'portage.util._eventloop.global_event_loop:global_event_loop',
'portage.util.ExtractKernelVersion:ExtractKernelVersion'
 )

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-09-07 Thread Zac Medico

commit: 6a3bdcc427c58112075c36cc0481c53215f12db4
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Sep  8 02:43:23 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Sep  8 02:44:46 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6a3bdcc4

binarytree.move_ent: fix path comparison prior to _movefile

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index e4393e06d..7e24589e5 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -531,7 +531,7 @@ class binarytree:
new_path = self.getname(mynewcpv)
self._pkg_paths[
self.dbapi._instance_key(mynewcpv)] = 
new_path[len(self.pkgdir)+1:]
-   if new_path != mytbz2:
+   if new_path != tbz2path:
self._ensure_dir(os.path.dirname(new_path))
_movefile(tbz2path, new_path, 
mysettings=self.settings)
self.inject(mynewcpv)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-08-08 Thread Zac Medico

commit: 656f8a7fcd2014c833e42282744c70a21e6c7e31
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Aug  8 23:51:52 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Aug  8 23:55:19 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=656f8a7f

treewalk: reset config for unmerge (bug 711174)

When cloning config instances for unmerge, call the reset
method in order to ensure that there is no unintended leakage
of variables which should not be shared. This fixes leakage
of the PORTAGE_LOG_FILE variable, which triggered log
corruption for FEATURES=compress-build-logs.

Bug: https://bugs.gentoo.org/711174
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 5799d94f2..3eee025ad 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -4013,7 +4013,9 @@ class dblink:
# since we need it to have private ${T} etc... 
for things
# like elog.
settings_clone = 
portage.config(clone=self.settings)
-   settings_clone.pop("PORTAGE_BUILDDIR_LOCKED", 
None)
+   # This reset ensures that there is no 
unintended leakage
+   # of variables which should not be shared.
+   settings_clone.reset()
settings_clone.setcpv(cur_cpv, 
mydb=self.vartree.dbapi)
if self._preserve_libs and "preserve-libs" in \

settings_clone["PORTAGE_RESTRICT"].split():

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-08-03 Thread Zac Medico

commit: f74cae3d377006af710f4b70cfbbf3391a126bb5
Author: Aaron Bauman  gentoo  org>
AuthorDate: Tue Aug  4 02:44:56 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Aug  4 03:13:01 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f74cae3d

lib/portage/dbapi/vartree.py: fix reimported

Signed-off-by: Aaron Bauman  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index fcf164e82..fbf455363 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1619,7 +1619,6 @@ class dblink:
At present this is implemented as a text backend in /var/db/pkg.
"""
 
-   import re
_normalize_needed = re.compile(r'//|^[^/]|./$|(^|/)\.\.?(/|$)')
 
_contents_re = re.compile(r'^(' + \

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-08-03 Thread Zac Medico

commit: ca6e92e3a08d0aad1dc551cfb1e380b843fc33b4
Author: Aaron Bauman  gentoo  org>
AuthorDate: Mon Aug  3 22:43:25 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Aug  3 23:28:04 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ca6e92e3

lib/portage/dbapi/virtual.py: fix whitespace

Signed-off-by: Aaron Bauman  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/virtual.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/virtual.py b/lib/portage/dbapi/virtual.py
index f62fc2a30..4ed0dea1b 100644
--- a/lib/portage/dbapi/virtual.py
+++ b/lib/portage/dbapi/virtual.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2013 Gentoo Foundation
+# Copyright 1998-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from portage.dbapi import dbapi
@@ -214,7 +214,7 @@ class fakedbapi(dbapi):
 
 class testdbapi:
"""A dbapi instance with completely fake functions to get by hitting 
disk
-   TODO(antarus): 
+   TODO(antarus):
This class really needs to be rewritten to have better stubs; but these 
work for now.
The dbapi classes themselves need unit tests...and that will be a lot 
of work.
"""

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-08-03 Thread Zac Medico

commit: 8af9f6d49363a1faeaa29a4fe00426dd2a1fabb4
Author: Aaron Bauman  gentoo  org>
AuthorDate: Mon Aug  3 22:43:14 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Aug  3 23:28:02 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8af9f6d4

lib/portage/dbapi/bintree.py: fix whitespace

Signed-off-by: Aaron Bauman  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 97be51121..59c265688 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1686,7 +1686,7 @@ class binarytree:
resume = True
writemsg(_("Resuming download of this tbz2, but it is 
possible that it is corrupt.\n"),
noiselevel=-1)
-   
+
mydest = os.path.dirname(self.getname(pkgname))
self._ensure_dir(mydest)
# urljoin doesn't work correctly with unrecognized protocols 
like sftp

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-08-03 Thread Zac Medico

commit: 3551cfb0be232df7f1a8d575181f8629ac7b4438
Author: Aaron Bauman  gentoo  org>
AuthorDate: Mon Aug  3 20:20:25 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Aug  3 21:25:52 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3551cfb0

lib/portage/dbapi/IndexedPortdb.py: drop unused-import

* Drop unused-import
* Update copyright

Signed-off-by: Aaron Bauman  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/IndexedPortdb.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/IndexedPortdb.py 
b/lib/portage/dbapi/IndexedPortdb.py
index 5f1cb5bd1..8d9ba89ff 100644
--- a/lib/portage/dbapi/IndexedPortdb.py
+++ b/lib/portage/dbapi/IndexedPortdb.py
@@ -1,4 +1,4 @@
-# Copyright 2014 Gentoo Foundation
+# Copyright 2014-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -15,7 +15,6 @@ from portage.cache.index.IndexStreamIterator import 
IndexStreamIterator
 from portage.cache.index.pkg_desc_index import \
pkg_desc_index_line_read, pkg_desc_index_node
 from portage.util.iterators.MultiIterGroupBy import MultiIterGroupBy
-from portage.versions import _pkg_str
 
 class IndexedPortdb:
"""

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-08-03 Thread Zac Medico

commit: 2af196d23ef2472432f0d8f29de9e4149978f8ef
Author: Aaron Bauman  gentoo  org>
AuthorDate: Mon Aug  3 20:20:22 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Aug  3 21:15:55 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2af196d2

lib/portage/dbapi/cpv_expand.py: drop unused-import

* Drop unused import
* Update copyright

Signed-off-by: Aaron Bauman  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/cpv_expand.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/portage/dbapi/cpv_expand.py b/lib/portage/dbapi/cpv_expand.py
index a1a91f554..eeec446d4 100644
--- a/lib/portage/dbapi/cpv_expand.py
+++ b/lib/portage/dbapi/cpv_expand.py
@@ -1,12 +1,10 @@
-# Copyright 2010-2013 Gentoo Foundation
+# Copyright 2010-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["cpv_expand"]
 
 import portage
 from portage.exception import AmbiguousPackageName
-from portage.localization import _
-from portage.util import writemsg
 from portage.versions import _pkgsplit
 
 def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-08-03 Thread Zac Medico

commit: c14bbdf7a32eca5d559c012682b6101d91d71bf2
Author: Aaron Bauman  gentoo  org>
AuthorDate: Mon Aug  3 19:05:59 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Aug  3 19:22:24 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c14bbdf7

lib/portage/dbapi/bintree.py: drop unused-import

Signed-off-by: Aaron Bauman  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/bintree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index e86fa5caa..97be51121 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -27,7 +27,7 @@ from portage.const import CACHE_PATH, 
SUPPORTED_XPAK_EXTENSIONS
 from portage.dbapi.virtual import fakedbapi
 from portage.dep import Atom, use_reduce, paren_enclose
 from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
-   ParseError, PermissionDenied, PortageException
+   ParseError, PortageException
 from portage.localization import _
 from portage.package.ebuild.profile_iuse import iter_iuse_vars
 from portage.util.futures import asyncio

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-07-22 Thread Zac Medico

commit: 309f2cc59e19eab05dfc2e86d1df4fc7ba1be93d
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Jul 22 20:10:30 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jul 22 20:12:16 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=309f2cc5

MergeProcess: handle cancelled future in _proc_join_done

If the future was cancelled, then avoid a possible ValueError
when accessing proc.exitcode.

Fixes: f587ebf3d492 ("MergeProcess: handle RETURNCODE_POSTINST_FAILURE in 
_proc_join_done")
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/_MergeProcess.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/_MergeProcess.py 
b/lib/portage/dbapi/_MergeProcess.py
index 85deedc18..e89b53555 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -199,7 +199,7 @@ class MergeProcess(ForkProcess):
"""
Extend _proc_join_done to react to RETURNCODE_POSTINST_FAILURE.
"""
-   if proc.exitcode == portage.const.RETURNCODE_POSTINST_FAILURE:
+   if not future.cancelled() and proc.exitcode == 
portage.const.RETURNCODE_POSTINST_FAILURE:
self.postinst_failure = True
self.returncode = os.EX_OK
super(MergeProcess, self)._proc_join_done(proc, future)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-07-22 Thread Zac Medico

commit: f587ebf3d4920b6c3cae3654918ba9cda6625a3e
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Jul 22 19:32:38 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jul 22 19:37:33 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f587ebf3

MergeProcess: handle RETURNCODE_POSTINST_FAILURE in _proc_join_done

Since ForkProcess now receives process exit status in the
_proc_join_done method instead of the _async_waitpid_cb method,
MergeProcess needs to handle RETURNCODE_POSTINST_FAILURE there
instead.

Fixes: 3561071e07ad ("MergeProcess: replace os.fork with 
multiprocessing.Process (bug 730192)")
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/_MergeProcess.py | 9 -
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/lib/portage/dbapi/_MergeProcess.py 
b/lib/portage/dbapi/_MergeProcess.py
index 6924c8b0e..85deedc18 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -195,15 +195,14 @@ class MergeProcess(ForkProcess):
prev_mtimes=self.prev_mtimes, 
counter=counter)
return rval
 
-   def _async_waitpid_cb(self, *args, **kwargs):
+   def _proc_join_done(self, proc, future):
"""
-   Override _async_waitpid_cb to perform cleanup that is
-   not necessarily idempotent.
+   Extend _proc_join_done to react to RETURNCODE_POSTINST_FAILURE.
"""
-   ForkProcess._async_waitpid_cb(self, *args, **kwargs)
-   if self.returncode == portage.const.RETURNCODE_POSTINST_FAILURE:
+   if proc.exitcode == portage.const.RETURNCODE_POSTINST_FAILURE:
self.postinst_failure = True
self.returncode = os.EX_OK
+   super(MergeProcess, self)._proc_join_done(proc, future)
 
def _unregister(self):
"""

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-07-22 Thread Zac Medico

commit: 3561071e07ad47db91bf0f2c2c2b02e2061b217c
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jul 19 06:31:37 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jul 22 16:55:33 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3561071e

MergeProcess: replace os.fork with multiprocessing.Process (bug 730192)

Fix the MergeProcess _spawn method to call the superclass _spawn
method, in order to replace os.fork with multiprocessing.Process,
promoting a healthy state for the forked interpreter.

Bug: https://bugs.gentoo.org/730192
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/_MergeProcess.py | 106 +++--
 1 file changed, 30 insertions(+), 76 deletions(-)

diff --git a/lib/portage/dbapi/_MergeProcess.py 
b/lib/portage/dbapi/_MergeProcess.py
index 274ef586f..6924c8b0e 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -3,9 +3,6 @@
 
 import io
 import platform
-import signal
-import sys
-import traceback
 
 import fcntl
 import portage
@@ -24,7 +21,7 @@ class MergeProcess(ForkProcess):
'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild',
'mydbapi', 'postinst_failure', 'prev_mtimes', 'unmerge',
'_elog_reader_fd',
-   '_buf', '_elog_keys', '_locked_vdb')
+   '_buf', '_counter', '_dblink', '_elog_keys', '_locked_vdb')
 
def _start(self):
# Portage should always call setcpv prior to this
@@ -103,8 +100,8 @@ class MergeProcess(ForkProcess):
 
def _spawn(self, args, fd_pipes, **kwargs):
"""
-   Fork a subprocess, apply local settings, and call
-   dblink.merge(). TODO: Share code with ForkProcess.
+   Extend the superclass _spawn method to perform some pre-fork and
+   post-fork actions.
"""
 
elog_reader_fd, elog_writer_fd = os.pipe()
@@ -132,57 +129,31 @@ class MergeProcess(ForkProcess):
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
-   counter = None
if not self.unmerge:
-   counter = self.vartree.dbapi.counter_tick()
-
-   parent_pid = os.getpid()
-   pid = None
-   try:
-   pid = os.fork()
-
-   if pid != 0:
-   if not isinstance(pid, int):
-   raise AssertionError(
-   "fork returned non-integer: %s" 
% (repr(pid),))
-
-   os.close(elog_writer_fd)
-   self._elog_reader_fd = elog_reader_fd
-   self._buf = ""
-   self._elog_keys = set()
-   # Discard messages which will be collected by 
the subprocess,
-   # in order to avoid duplicates (bug #446136).
-   
portage.elog.messages.collect_messages(key=mylink.mycpv)
-
-   # invalidate relevant vardbapi caches
-   if self.vartree.dbapi._categories is not None:
-   self.vartree.dbapi._categories = None
-   self.vartree.dbapi._pkgs_changed = True
-   self.vartree.dbapi._clear_pkg_cache(mylink)
-
-   return [pid]
-
-   os.close(elog_reader_fd)
-
-   # Use default signal handlers in order to avoid problems
-   # killing subprocesses as reported in bug #353239.
-   signal.signal(signal.SIGINT, signal.SIG_DFL)
-   signal.signal(signal.SIGTERM, signal.SIG_DFL)
-
-   # Unregister SIGCHLD handler and wakeup_fd for the 
parent
-   # process's event loop (bug 655656).
-   signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-   try:
-   wakeup_fd = signal.set_wakeup_fd(-1)
-   if wakeup_fd > 0:
-   os.close(wakeup_fd)
-   except (ValueError, OSError):
-   pass
-
-   portage.locks._close_fds()
-   # We don't exec, so use close_fds=False
-   # (see _setup_pipes docstring).
-   portage.process._setup_pipes(fd_pipes, close_fds=False)
+   self._counter = self.vartree.dbapi.counter_tick()
+
+   self._dblink = mylink
+   self._elog_reader_fd = elog_reader_fd
+   pids =

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, bin/, lib/portage/_emirrordist/, lib/portage/util/_dyn_libs/, ...

2020-07-17 Thread Michał Górny

commit: 55a6bfdd9d3e2b8d0a59f5579de962fb82dde8ca
Author: Michał Górny  gentoo  org>
AuthorDate: Fri Jul 17 04:34:00 2020 +
Commit: Michał Górny  gentoo  org>
CommitDate: Fri Jul 17 04:39:07 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=55a6bfdd

Remove support code for Python < 3.3

Signed-off-by: Michał Górny  gentoo.org>

 bin/chpathtool.py  |  8 +--
 bin/doins.py   | 14 +
 lib/portage/_emirrordist/FetchTask.py  | 13 ++---
 lib/portage/dbapi/vartree.py   | 25 ++---
 lib/portage/util/_compare_files.py | 22 +++-
 .../util/_dyn_libs/PreservedLibsRegistry.py|  7 +--
 lib/portage/util/movefile.py   | 60 ++
 lib/portage/util/mtimedb.py|  7 +--
 8 files changed, 39 insertions(+), 117 deletions(-)

diff --git a/bin/chpathtool.py b/bin/chpathtool.py
index fbd18b987..c036046ae 100755
--- a/bin/chpathtool.py
+++ b/bin/chpathtool.py
@@ -128,12 +128,8 @@ def chpath_inplace(filename, is_text_file, old, new):
 
f.close()
if modified:
-   if sys.hexversion >= 0x303:
-   orig_mtime = orig_stat.st_mtime_ns
-   os.utime(filename, ns=(orig_mtime, orig_mtime))
-   else:
-   orig_mtime = orig_stat[stat.ST_MTIME]
-   os.utime(filename, (orig_mtime, orig_mtime))
+   orig_mtime = orig_stat.st_mtime_ns
+   os.utime(filename, ns=(orig_mtime, orig_mtime))
return modified
 
 def chpath_inplace_symlink(filename, st, old, new):

diff --git a/bin/doins.py b/bin/doins.py
index 98dc4f810..a08e3f8c9 100644
--- a/bin/doins.py
+++ b/bin/doins.py
@@ -110,10 +110,6 @@ def _parse_install_options(
parser.add_argument('-p', '--preserve-timestamps', action='store_true')
split_options = shlex.split(options)
namespace, remaining = parser.parse_known_args(split_options)
-   if namespace.preserve_timestamps and sys.version_info < (3, 3):
-   # -p is not supported in this case, since timestamps cannot
-   # be preserved with full precision
-   remaining.append('-p')
# Because parsing '--mode' option is partially supported. If unknown
# arg for --mode is passed, namespace.mode is set to None.
if remaining or namespace.mode is None:
@@ -151,15 +147,7 @@ def _set_timestamps(source_stat, dest):
source_stat: stat result for the source file.
dest: path to the dest file.
"""
-   os.utime(dest, (source_stat.st_atime, source_stat.st_mtime))
-
-
-if sys.version_info >= (3, 3):
-   def _set_timestamps_ns(source_stat, dest):
-   os.utime(dest, ns=(source_stat.st_atime_ns, 
source_stat.st_mtime_ns))
-
-   _set_timestamps_ns.__doc__ = _set_timestamps.__doc__
-   _set_timestamps = _set_timestamps_ns
+   os.utime(dest, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns))
 
 
 class _InsInProcessInstallRunner(object):

diff --git a/lib/portage/_emirrordist/FetchTask.py 
b/lib/portage/_emirrordist/FetchTask.py
index 322de79ba..a1ba58822 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Gentoo Authors
+# Copyright 2013-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import division
@@ -385,14 +385,9 @@ class FetchTask(CompositeTask):
# Apply the timestamp from the source file, but
# just rely on umask for permissions.
try:
-   if sys.hexversion >= 0x303:
-   os.utime(copier.dest_path,
-   
ns=(self._current_stat.st_mtime_ns,
-   self._current_stat.st_mtime_ns))
-   else:
-   os.utime(copier.dest_path,
-   
(self._current_stat[stat.ST_MTIME],
-   
self._current_stat[stat.ST_MTIME]))
+   os.utime(copier.dest_path,
+   ns=(self._current_stat.st_mtime_ns,
+   self._current_stat.st_mtime_ns))
except OSError as e:
msg = "%s %s utime failed unexpectedly: %s" % \
(self.distfile, current_mirror.name, e)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index abb0a9308..2e29b25e5 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -435,10 +435,7 @@ class

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/_emerge/, lib/portage/util/_async/, ...

2020-06-22 Thread Zac Medico

commit: dd69ce742c62b9515cf7ae37e46bcf7f178777db
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Mar  1 02:17:52 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Jun 23 02:13:06 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dd69ce74

Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)

This variable specifies a command that filters build log output to a
log file. The plan is to extend this to support a separate filter for
tty output in the future.

In order to enable the EbuildPhase class to write elog messages to
the build log with PORTAGE_LOG_FILTER_FILE_CMD support, convert its
_elog method to a coroutine, and add a SchedulerInterface async_output
method for it to use.

Use a new BuildLogger class to manage log output (with or without a
filter command), with compression support provided by PipeLogger.
BuildLogger has a stdin property which provides access to a writable
binary file stream (refers to a pipe) that log content is written to.

Bug: https://bugs.gentoo.org/709746
Reviewed-by: Brian Dolbec  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/_emerge/AbstractEbuildProcess.py   |   3 +-
 lib/_emerge/BinpkgFetcher.py   |   3 +-
 lib/_emerge/EbuildFetcher.py   |   3 +-
 lib/_emerge/EbuildPhase.py |  47 +++--
 lib/_emerge/SpawnProcess.py|  58 ---
 lib/portage/dbapi/_MergeProcess.py |   3 +-
 .../package/ebuild/_config/special_env_vars.py |   8 +-
 lib/portage/util/_async/BuildLogger.py | 109 +
 lib/portage/util/_async/SchedulerInterface.py  |  32 +-
 man/make.conf.5|   7 +-
 10 files changed, 243 insertions(+), 30 deletions(-)

diff --git a/lib/_emerge/AbstractEbuildProcess.py 
b/lib/_emerge/AbstractEbuildProcess.py
index 1c1955cfe..ae1aae55f 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2019 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -196,6 +196,7 @@ class AbstractEbuildProcess(SpawnProcess):
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
 
+   self.log_filter_file = 
self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
try:
SpawnProcess._start(self)
finally:

diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 36d027de3..2e5861cc1 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -158,6 +158,7 @@ class _BinpkgFetcherProcess(SpawnProcess):
self.env = fetch_env
if settings.selinux_enabled():
self._selinux_type = settings["PORTAGE_FETCH_T"]
+   self.log_filter_file = 
settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
SpawnProcess._start(self)
 
def _pipe(self, fd_pipes):

diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py
index 1e40994fb..55349c33c 100644
--- a/lib/_emerge/EbuildFetcher.py
+++ b/lib/_emerge/EbuildFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import copy
@@ -225,6 +225,7 @@ class _EbuildFetcherProcess(ForkProcess):
settings["NOCOLOR"] = nocolor
 
self._settings = settings
+   self.log_filter_file = 
settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
ForkProcess._start(self)
 
# Free settings now since it's no longer needed in

diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 477e0ba97..ddb3dc719 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -26,6 +26,8 @@ from portage.package.ebuild.prepare_build_dirs import 
(_prepare_workdir,
 from portage.util.futures.compat_coroutine import coroutine
 from portage.util import writemsg
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._async.BuildLogger import BuildLogger
+from portage.util.futures import asyncio
 from portage.util.futures.executor.fork import ForkExecutor
 
 try:
@@ -69,6 +71,11 @@ class EbuildPhase(CompositeTask):
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
 
def _start(self):
+   future = asyncio.ensure_future(self._async_start(), 
loop=self.scheduler)
+   self._start_task(AsyncTaskFuture(future=future), 
self._async_start_exit)
+
+   @coroutine
+

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-06-06 Thread Zac Medico

commit: 976c1133100f6da81cd8d6e13f8a723a9fc7cd85
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jun  7 02:25:11 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jun  7 02:53:27 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=976c1133

_better_cache._scan_cat: avoid stat calls (bug 725934)

When processing category listdir results, do not use os.path.isdir to
identify packages, in order to avoid unecessary stat calls. Instead,
use the Atom class to validate package names. This may cause creation
of some cache entries for non-packages, but it will not do any harm
since these entries will never be accessed via the __getitem__ method.

Bug: https://bugs.gentoo.org/725934
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/porttree.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 08af17bcd..ed992d1e2 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -164,8 +164,13 @@ class _better_cache(object):
raise
continue
for p in pkg_list:
-   if os.path.isdir(cat_dir + "/" + p):
-   self._items[cat + "/" + p].append(repo)
+   try:
+   atom = Atom("%s/%s" % (cat, p))
+   except InvalidAtom:
+   continue
+   if atom != atom.cp:
+   continue
+   self._items[atom.cp].append(repo)
self._scanned_cats.add(cat)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-02-20 Thread Zac Medico

commit: 3de00842d69669a15adaf9e81b0007a7052da5d0
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Feb 20 09:51:34 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Feb 20 09:52:33 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3de00842

async_fetch_map: handle _parse_uri_map exception (bug 710130)

If _parse_uri_map raises an exception then raise it to the
caller via the returned future.

Bug: https://bugs.gentoo.org/710130
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/porttree.py | 9 ++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 4bb396a6c..08af17bcd 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2019 Gentoo Authors
+# Copyright 1998-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -804,8 +804,11 @@ class portdbapi(dbapi):
(mypkg, eapi)))
return
 
-   result.set_result(_parse_uri_map(mypkg,
-   {'EAPI':eapi,'SRC_URI':myuris}, use=useflags))
+   try:
+   result.set_result(_parse_uri_map(mypkg,
+   {'EAPI':eapi,'SRC_URI':myuris}, 
use=useflags))
+   except Exception as e:
+   result.set_exception(e)
 
aux_get_future = self.async_aux_get(
mypkg, ["EAPI", "SRC_URI"], mytree=mytree, loop=loop)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2020-02-02 Thread Zac Medico

commit: 6d8104e665b55174980cd88695f0f956935507b7
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Feb  3 03:01:06 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Feb  3 03:02:02 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6d8104e6

sphinx-build: remove broken lazyimport for _spawn_phase

Fixes: d318bcce356b ("vartree: add missing _merge_unicode_error import")
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 29843656f..050366528 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -23,7 +23,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.locks:lockdir,unlockdir,lockfile,unlockfile',
'portage.output:bold,colorize',
'portage.package.ebuild.doebuild:doebuild_environment,' + \
-   '_merge_unicode_error', '_spawn_phase',
+   '_merge_unicode_error',
'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
'portage.package.ebuild._ipc.QueryCommand:QueryCommand',
'portage.process:find_binary',

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2019-08-23 Thread Zac Medico

commit: f90400eedc6a8788878c050880db564800c825d2
Author: Zac Medico  gentoo  org>
AuthorDate: Fri Aug 23 20:00:15 2019 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Aug 24 02:41:35 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f90400ee

preserve-libs: ignore dropped non-soname symlink (bug 692698)

Fix the dblink _find_libs_to_preserve method to ignore a dropped
non-soname symlink. For example, pam-1.3.1-r1 drops the non-soname
symlink named libpam_misc.so, and we don't want this to trigger
unnecessary preservation of the corresponding library, since the
corresponding libpam_misc.so.0 soname symlink and the hardlink
that it references are still provided by pam-1.3.1-r1.

Bug: https://bugs.gentoo.org/692698
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py | 15 +++
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 4f069474b..fa1e1523c 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -3133,10 +3133,6 @@ class dblink(object):
os = portage.os
 
f = f_abs[root_len:]
-   if not unmerge and self.isowner(f):
-   # We have an indentically named replacement 
file,
-   # so we don't try to preserve the old copy.
-   continue
try:
consumers = linkmap.findConsumers(f,

exclude_providers=(installed_instance.isowner,))
@@ -3184,16 +3180,27 @@ class dblink(object):
hardlinks = set()
soname_symlinks = set()
soname = 
linkmap.getSoname(next(iter(preserve_node.alt_paths)))
+   have_replacement_soname_link = False
+   have_replacement_hardlink = False
for f in preserve_node.alt_paths:
f_abs = os.path.join(root, f.lstrip(os.sep))
try:
if 
stat.S_ISREG(os.lstat(f_abs).st_mode):
hardlinks.add(f)
+   if not unmerge and 
self.isowner(f):
+   
have_replacement_hardlink = True
+   if os.path.basename(f) 
== soname:
+   
have_replacement_soname_link = True
elif os.path.basename(f) == soname:
soname_symlinks.add(f)
+   if not unmerge and 
self.isowner(f):
+   
have_replacement_soname_link = True
except OSError:
pass
 
+   if have_replacement_hardlink and 
have_replacement_soname_link:
+   continue
+
if hardlinks:
preserve_paths.update(hardlinks)
preserve_paths.update(soname_symlinks)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/

2019-08-11 Thread Zac Medico

commit: 0b7eda500a0dcb98a67f33bf9ef25b202b358986
Author: Arfrever Frehtes Taifersar Arahesis  Apache  Org>
AuthorDate: Wed Aug  7 17:06:11 2019 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Aug 11 19:11:47 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0b7eda50

dblink._collision_protect: Detect internal collisions.

Implement detection of internal collisions (between files of the same package,
located in separate directories in the installation image (${D}) corresponding
to merged directories in the target filesystem (${ROOT})).

This provides protection against overwriting some files when performing merging
of files from ${D} to ${ROOT} in some filesystem layouts (such as /-merged 
layout
or /usr-merged layout).

Internal collisions between identical files are silently ignored.

Bug: https://bugs.gentoo.org/690484
Signed-off-by: Arfrever Frehtes Taifersar Arahesis  Apache.Org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/vartree.py   |  55 ++--
 lib/portage/util/_compare_files.py | 103 +
 2 files changed, 154 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index e2fce7736..4f069474b 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -30,6 +30,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \

'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \
'grabdict,normalize_path,new_protect_filename',
+   'portage.util._compare_files:compare_files',
'portage.util.digraph:digraph',
'portage.util.env_update:env_update',
'portage.util.install_mask:install_mask_dir,InstallMask',
@@ -87,6 +88,7 @@ import io
 from itertools import chain
 import logging
 import os as _os
+import operator
 import platform
 import pwd
 import re
@@ -3418,6 +3420,8 @@ class dblink(object):
 
os = _os_merge
 
+   real_relative_paths = {}
+
collision_ignore = []
for x in portage.util.shlex_split(
self.settings.get("COLLISION_IGNORE", "")):
@@ -3469,8 +3473,13 @@ class dblink(object):
previous = current
progress_shown = True
 
-   dest_path = normalize_path(
-   os.path.join(destroot, 
f.lstrip(os.path.sep)))
+   dest_path = 
normalize_path(os.path.join(destroot, f.lstrip(os.path.sep)))
+
+   # Relative path with symbolic links resolved 
only in parent directories
+   real_relative_path = 
os.path.join(os.path.realpath(os.path.dirname(dest_path)),
+   
os.path.basename(dest_path))[len(destroot):]
+
+   
real_relative_paths.setdefault(real_relative_path, 
[]).append(f.lstrip(os.path.sep))
 
parent = os.path.dirname(dest_path)
if parent not in dirs:
@@ -3556,9 +3565,24 @@ class dblink(object):
break
if stopmerge:
collisions.append(f)
+
+   internal_collisions = {}
+   for real_relative_path, files in 
real_relative_paths.items():
+   # Detect internal collisions between 
non-identical files.
+   if len(files) >= 2:
+   files.sort()
+   for i in range(len(files) - 1):
+   file1 = 
normalize_path(os.path.join(srcroot, files[i]))
+   file2 = 
normalize_path(os.path.join(srcroot, files[i+1]))
+   # Compare files, ignoring 
differences in times.
+   differences = 
compare_files(file1, file2, skipped_types=("atime", "mtime", "ctime"))
+   if differences:
+   
internal_collisions.setdefault(real_relative_path, {})[(files[i], files[i+1])] 
= differences
+
if progress_shown:
showMessage(_("100% done\n"))
-   return collisions, dirs_ro, symlink_collisions, 
plib_collisions
+
+   return collisions, internal_collisions, dirs_ro, 
symlink_collisions, plib_collisions
 
def _lstat_inode_map(self, path_iter):
"""
@@ -4081,7 +4105,7 @@ class

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2019-06-20 Thread Zac Medico

commit: 611a53a6943bf25ac8c1d909e34e4b37c8873e83
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Jun 20 01:37:53 2019 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Jun 20 02:21:36 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=611a53a6

portdbapi.getfetchsizes: use .__download__ suffix

If the file does not exist locally or the size is not correct,
use the file with the .__download__ suffix to compute the
remaining_size to fetch.

Fixes: ebbde237d33e ("fetch: atomic downloads (bug 175612)")
Bug: https://bugs.gentoo.org/688124
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/porttree.py | 12 
 1 file changed, 12 insertions(+)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index edff0c2f2..e4b688b9a 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -14,6 +14,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.dbapi.dep_expand:dep_expand',
'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot',
'portage.package.ebuild.doebuild:doebuild',
+   'portage.package.ebuild.fetch:_download_suffix',
'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
'portage.util.listdir:listdir',

'portage.versions:best,catsplit,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str',
@@ -844,6 +845,17 @@ class portdbapi(dbapi):
mystat = os.stat(file_path)
except OSError:
pass
+   else:
+   if mystat.st_size != fetch_size:
+   # Use file with _download_suffix 
instead.
+   mystat = None
+
+   if mystat is None:
+   try:
+   mystat = os.stat(file_path + 
_download_suffix)
+   except OSError:
+   pass
+
if mystat is None:
existing_size = 0
ro_distdirs = 
self.settings.get("PORTAGE_RO_DISTDIRS")

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/futures/_asyncio/

2019-05-12 Thread Zac Medico

commit: 00340dfa7d85144aa975079d6eeebaa035178208
Author: Zac Medico  gentoo  org>
AuthorDate: Sun May 12 06:51:49 2019 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun May 12 07:02:03 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=00340dfa

portdbapi._event_loop: split out _safe_loop function

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/porttree.py | 10 +-
 lib/portage/util/futures/_asyncio/__init__.py | 18 ++
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 7a28f5876..edff0c2f2 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -34,7 +34,6 @@ from portage import eclass_cache, \
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
-from portage.util._eventloop.EventLoop import EventLoop
 from portage.util.futures import asyncio
 from portage.util.futures.compat_coroutine import coroutine, coroutine_return
 from portage.util.futures.iter_completed import iter_gather
@@ -346,14 +345,7 @@ class portdbapi(dbapi):
 
@property
def _event_loop(self):
-   if portage._internal_caller:
-   # For internal portage usage, asyncio._wrap_loop() is 
safe.
-   return asyncio._wrap_loop()
-   else:
-   # For external API consumers, use a local EventLoop, 
since
-   # we don't want to assume that it's safe to override the
-   # global SIGCHLD handler.
-   return EventLoop(main=False)
+   return asyncio._safe_loop()
 
def _create_pregen_cache(self, tree):
conf = self.repositories.get_repo_for_location(tree)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index 2a637624d..e77c7a690 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -37,6 +37,7 @@ import portage
 portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures.unix_events:_PortageEventLoopPolicy',
'portage.util.futures:compat_coroutine@_compat_coroutine',
+   'portage.util._eventloop.EventLoop:EventLoop@_EventLoop',
 )
 from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as 
_AsyncioEventLoop
 from portage.util._eventloop.global_event_loop import (
@@ -250,3 +251,20 @@ if _asyncio_enabled:
loop = loop or _global_event_loop()
return (loop if hasattr(loop, '_asyncio_wrapper')
else _AsyncioEventLoop(loop=loop))
+
+
+def _safe_loop():
+   """
+   Return an event loop that's safe to use within the current context.
+   For portage internal callers, this returns a globally shared event
+   loop instance. For external API consumers, this constructs a
+   temporary event loop instance that's safe to use in a non-main
+   thread (it does not override the global SIGCHLD handler).
+
+   @rtype: asyncio.AbstractEventLoop (or compatible)
+   @return: event loop instance
+   """
+   if portage._internal_caller:
+   return _wrap_loop()
+   else:
+   return _EventLoop(main=False)

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2019-05-11 Thread Zac Medico

commit: fbebef9677d63db70f1c68b197e58b041ec6ac61
Author: Zac Medico  gentoo  org>
AuthorDate: Thu May  9 20:10:46 2019 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat May 11 20:55:49 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fbebef96

repoman: check IUSE in _match_use for USE defaults (bug 685482)

This corrects a problem triggered with USE defaults where repoman
returns an incorrect negative match for an atom with USE defaults.
For example, it triggered this dependency.bad error:

RepoMan scours the neighborhood...
  dependency.bad [fatal]2
   dev-libs/libxml2/libxml2-2.9.9-r1.ebuild: DEPEND: 
~riscv(default/linux/riscv/17.0/rv64gc)
[ 
'>=dev-lang/python-exec-2:=[python_targets_python2_7(-)?,python_targets_python3_5(-)?,python_targets_python3_6(-)?,python_targets_python3_7(-)?,-python_single_target_python2_7(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-),-python_single_target_python3_7(-)]']
   dev-libs/libxml2/libxml2-2.9.9-r1.ebuild: RDEPEND: 
~riscv(default/linux/riscv/17.0/rv64gc)
[ 
'>=dev-lang/python-exec-2:=[python_targets_python2_7(-)?,python_targets_python3_5(-)?,python_targets_python3_6(-)?,python_targets_python3_7(-)?,-python_single_target_python2_7(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-),-python_single_target_python3_7(-)]']

State in the _match_use method looked like this:

(Pdb) atom
'>=dev-lang/python-exec-2:=[python_targets_python3_7(-),-python_single_target_python2_7(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-),-python_single_target_python3_7(-)]'
(Pdb) useforce
frozenset({'python_targets_pypy', 'python_targets_pypy3', 
'python_targets_python3_5', 'big-endian', 'userland_GNU', 
'python_targets_python3_7', 'riscv', 'python_single_target_python3_7', 
'python_targets_python2_7', 'elibc_glibc', 'python_targets_python3_6', 
'abi_riscv_lp64d', 'kernel_linux', 'python_targets_jython2_7'})
(Pdb) atom.use.disabled
frozenset({'python_single_target_python3_6', 'python_single_target_python2_7', 
'python_single_target_python3_7', 'python_single_target_python3_5'})
(Pdb) atom.use.missing_disabled
frozenset({'python_single_target_python2_7', 'python_targets_python3_5', 
'python_targets_python3_7', 'python_single_target_python3_5', 
'python_single_target_python3_7', 'python_targets_python2_7', 
'python_targets_python3_6', 'python_single_target_python3_6'})

Bug: https://bugs.gentoo.org/685482
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 6fca6090c..80f8a689f 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -314,12 +314,12 @@ class dbapi(object):
# Check masked and forced flags for repoman.
usemask = self.settings._getUseMask(pkg,
stable=self.settings._parent_stable)
-   if any(x in usemask for x in atom.use.enabled):
+   if any(x in usemask and iuse.get_real_flag(x) 
is not None for x in atom.use.enabled):
return False
 
useforce = self.settings._getUseForce(pkg,
stable=self.settings._parent_stable)
-   if any(x in useforce and x not in usemask
+   if any(x in useforce and x not in usemask and 
iuse.get_real_flag(x) is not None
for x in atom.use.disabled):
return False

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/futures/executor/, ...

2019-02-16 Thread Robin H. Johnson

commit: 3888e16314afa9107aa0f2d9dce864b85246822c
Author: Robin H. Johnson  gentoo  org>
AuthorDate: Sat Feb 16 06:17:12 2019 +
Commit: Robin H. Johnson  gentoo  org>
CommitDate: Sat Feb 16 06:17:12 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3888e163

Replace multiprocessing.cpu_count with portage.util.cpuinfo.get_cpu_count

portage.util.cpuinfo.get_cpu_count was only used in one spot before, and
other call-sites just used multiprocessing.cpu_count() directly.

Replace all multiprocessing.cpu_count() calls with get_cpu_count() in
portage.util.cpuinfo, to ensure consistency in CPU calculation.

Signed-off-by: Robin H. Johnson  gentoo.org>

 lib/portage/dbapi/porttree.py  |  4 ++--
 lib/portage/util/futures/executor/fork.py  |  4 ++--
 lib/portage/util/futures/iter_completed.py | 18 +-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 2ff3e1b34..64a5f3681 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1471,11 +1471,11 @@ def _async_manifest_fetchlist(portdb, repo_config, cp, 
cpv_list=None,
@param cpv_list: list of ebuild cpv values for a Manifest
@type cpv_list: list
@param max_jobs: max number of futures to process concurrently (default
-   is multiprocessing.cpu_count())
+   is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
-   is multiprocessing.cpu_count())
+   is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop

diff --git a/lib/portage/util/futures/executor/fork.py 
b/lib/portage/util/futures/executor/fork.py
index 72844403c..add7b3c9e 100644
--- a/lib/portage/util/futures/executor/fork.py
+++ b/lib/portage/util/futures/executor/fork.py
@@ -7,13 +7,13 @@ __all__ = (
 
 import collections
 import functools
-import multiprocessing
 import os
 import sys
 import traceback
 
 from portage.util._async.AsyncFunction import AsyncFunction
 from portage.util.futures import asyncio
+from portage.util.cpuinfo import get_cpu_count
 
 
 class ForkExecutor(object):
@@ -24,7 +24,7 @@ class ForkExecutor(object):
This is entirely driven by an event loop.
"""
def __init__(self, max_workers=None, loop=None):
-   self._max_workers = max_workers or multiprocessing.cpu_count()
+   self._max_workers = max_workers or get_cpu_count()
self._loop = asyncio._wrap_loop(loop)
self._submit_queue = collections.deque()
self._running_tasks = {}

diff --git a/lib/portage/util/futures/iter_completed.py 
b/lib/portage/util/futures/iter_completed.py
index 31b5e0c78..4c48ea0fe 100644
--- a/lib/portage/util/futures/iter_completed.py
+++ b/lib/portage/util/futures/iter_completed.py
@@ -2,11 +2,11 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
-import multiprocessing
 
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
 from portage.util._async.TaskScheduler import TaskScheduler
 from portage.util.futures import asyncio
+from portage.util.cpuinfo import get_cpu_count
 
 
 def iter_completed(futures, max_jobs=None, max_load=None, loop=None):
@@ -18,11 +18,11 @@ def iter_completed(futures, max_jobs=None, max_load=None, 
loop=None):
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
-   is multiprocessing.cpu_count())
+   is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
-   is multiprocessing.cpu_count())
+   is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float
@param loop: event loop
@type loop: EventLoop
@@ -47,11 +47,11 @@ def async_iter_completed(futures, max_jobs=None, 
max_load=None, loop=None):
@param futures: iterator of asyncio.Future (or compatible)
@type futures: iterator
@param max_jobs: max number of futures to process concurrently (default
-   is multiprocessing.cpu_count())
+   is portage.util.cpuinfo.get_cpu_count())
@type max_jobs: int
@param max_load: max load allowed when scheduling a new future,
otherwise schedule no more than 1 future at a time (default
-   is multiprocessing.cpu_count())
+   is portage.util.cpuinfo.get_cpu_count())
@type max_load: int or float

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2019-01-19 Thread Zac Medico

commit: bbe00f515b80be3d6e279e4033f1cbc776a7fe27
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jan 20 06:39:39 2019 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jan 20 06:53:34 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bbe00f51

portagetree: getname and portroot DeprecationWarning

Bug: https://bugs.gentoo.org/668164
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/porttree.py | 16 ++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 76b7967f7..2ff3e1b34 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1339,10 +1339,18 @@ class portagetree(object):
" constructor is unused",
DeprecationWarning, stacklevel=2)
 
-   self.portroot = settings["PORTDIR"]
self.__virtual = virtual
self.dbapi = portdbapi(mysettings=settings)
 
+   @property
+   def portroot(self):
+   """Deprecated. Use the portdbapi getRepositoryPath method 
instead."""
+   warnings.warn("The portroot attribute of "
+   "portage.dbapi.porttree.portagetree is deprecated. Use 
the "
+   "portdbapi getRepositoryPath method instead.",
+   DeprecationWarning, stacklevel=3)
+   return self.settings['PORTDIR']
+
@property
def root(self):
warnings.warn("The root attribute of " + \
@@ -1383,7 +1391,11 @@ class portagetree(object):
return self.dbapi.cp_all()
 
def getname(self, pkgname):
-   "returns file location for this particular package (DEPRECATED)"
+   """Deprecated. Use the portdbapi findname method instead."""
+   warnings.warn("The getname method of "
+   "portage.dbapi.porttree.portagetree is deprecated. "
+   "Use the portdbapi findname method instead.",
+   DeprecationWarning, stacklevel=2)
if not pkgname:
return ""
mysplit = pkgname.split("/")

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2019-01-11 Thread Fabian Groffen

commit: cfa915d0d575379df4b9f17fd2db3594155861ca
Author: Fabian Groffen  gentoo  org>
AuthorDate: Mon Jan  7 14:19:15 2019 +
Commit: Fabian Groffen  gentoo  org>
CommitDate: Fri Jan 11 10:14:22 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cfa915d0

collision_protect: use dynamic report interval

The reporting of files remaining can look somewhat odd since the report
interval is hardcoded to be per 1000 objects.  Adjust this interval to
be time based.  This means that modern (fast) machines likely will never
see the countdown messages at all.  On slow setups the message will be
informative that there is progress, albeit rather slowly.  While at it,
report percentage done.

Output before this patch:

 * checking 6158 files for package collisions
5158 files remaining ...
4158 files remaining ...
3158 files remaining ...
2158 files remaining ...
1158 files remaining ...
158 files remaining ...

Possible output after this patch on a slower machine:

 * checking 6158 files for package collisions
 48% done,  3145 files remaining ...
 96% done,   192 files remaining ...
100% done

Signed-off-by: Fabian Groffen  gentoo.org>

 lib/portage/dbapi/vartree.py | 15 +--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 9febf0c71..63389f9a3 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -35,6 +35,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.install_mask:install_mask_dir,InstallMask',
'portage.util.listdir:dircache,listdir',
'portage.util.movefile:movefile',
+   'portage.util.monotonic:monotonic',
'portage.util.path:first_existing,iter_parents',
'portage.util.writeable_check:get_ro_checker',
'portage.util._xattr:xattr',
@@ -3453,13 +3454,21 @@ class dblink(object):
symlink_collisions = []
destroot = self.settings['ROOT']
totfiles = len(file_list) + len(symlink_list)
+   previous = monotonic()
+   progress_shown = False
+   report_interval = 1.7  # seconds
+   falign = len("%d" % totfiles)
showMessage(_(" %s checking %d files for package 
collisions\n") % \
(colorize("GOOD", "*"), totfiles))
for i, (f, f_type) in enumerate(chain(
((f, "reg") for f in file_list),
((f, "sym") for f in symlink_list))):
-   if i % 1000 == 0 and i != 0:
-   showMessage(_("%d files remaining 
...\n") % (totfiles - i))
+   current = monotonic()
+   if current - previous > report_interval:
+   showMessage(_("%3d%% done,  %*d files 
remaining ...\n") %
+   (i * 100 / totfiles, 
falign, totfiles - i))
+   previous = current
+   progress_shown = True
 
dest_path = normalize_path(
os.path.join(destroot, 
f.lstrip(os.path.sep)))
@@ -3548,6 +3557,8 @@ class dblink(object):
break
if stopmerge:
collisions.append(f)
+   if progress_shown:
+   showMessage(_("100% done\n"))
return collisions, dirs_ro, symlink_collisions, 
plib_collisions
 
def _lstat_inode_map(self, path_iter):

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2018-09-24 Thread Zac Medico

commit: ebd2c386f5ae2aced4c3ea05dacffdb99bd0bf5b
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Sep 24 03:04:13 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Sep 24 06:45:32 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ebd2c386

portdbapi: add async_xmatch method (bug 666940)

Add an async_xmatch method, and use it to implement the synchronous
xmatch method. Deprecate unused xmatch method parameters. Use the
compat_coroutine decorator for backward compatibility with python2.7
(via PEP 342 enhanced generators).

Bug: https://bugs.gentoo.org/666940
Suggested-by: Daniel Robbins  funtoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/porttree.py | 60 +++
 1 file changed, 49 insertions(+), 11 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 56955ec34..76b7967f7 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -37,6 +37,7 @@ from portage import _unicode_encode
 from portage import OrderedDict
 from portage.util._eventloop.EventLoop import EventLoop
 from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
 from portage.util.futures.iter_completed import iter_gather
 from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
 
@@ -1055,8 +1056,20 @@ class portdbapi(dbapi):
self._better_cache = None
self.frozen = 0
 
-   def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None):
-   "caching match function; very trick stuff"
+   def xmatch(self, level, origdep, mydep=DeprecationWarning,
+   mykey=DeprecationWarning, mylist=DeprecationWarning):
+   """
+   Caching match function.
+
+   @param level: xmatch level (bestmatch-visible, 
match-all-cpv-only
+   match-allmatch-visible, minimum-all, 
minimum-all-ignore-profile,
+   or minimum-visible)
+   @type level: str
+   @param origdep: dependency to match (may omit category)
+   @type origdep: portage.dep.Atom or str
+   @return: match result(s)
+   @rtype: _pkg_str or list of _pkg_str (depends on level)
+   """
if level == "list-visible":
level = "match-visible"
warnings.warn("The 'list-visible' mode of "
@@ -1064,21 +1077,46 @@ class portdbapi(dbapi):
"has been renamed to match-visible",
DeprecationWarning, stacklevel=2)
 
-   if mydep is None:
-   #this stuff only runs on first call of xmatch()
-   #create mydep, mykey from origdep
-   mydep = dep_expand(origdep, mydb=self, 
settings=self.settings)
-   mykey = mydep.cp
+   if mydep is not DeprecationWarning:
+   warnings.warn("The 'mydep' parameter of "
+   "portage.dbapi.porttree.portdbapi.xmatch"
+   " is deprecated and ignored",
+   DeprecationWarning, stacklevel=2)
+
+   loop = self._event_loop
+   return loop.run_until_complete(
+   self.async_xmatch(level, origdep, loop=loop))
+
+   @coroutine
+   def async_xmatch(self, level, origdep, loop=None):
+   """
+   Asynchronous form of xmatch.
+
+   @param level: xmatch level (bestmatch-visible, 
match-all-cpv-only
+   match-allmatch-visible, minimum-all, 
minimum-all-ignore-profile,
+   or minimum-visible)
+   @type level: str
+   @param origdep: dependency to match (may omit category)
+   @type origdep: portage.dep.Atom or str
+   @param loop: event loop (defaults to global event loop)
+   @type loop: EventLoop
+   @return: match result(s)
+   @rtype: asyncio.Future (or compatible), which results in a 
_pkg_str
+   or list of _pkg_str (depends on level)
+   """
+   mydep = dep_expand(origdep, mydb=self, settings=self.settings)
+   mykey = mydep.cp
 
#if no updates are being made to the tree, we can consult our 
xcache...
cache_key = None
if self.frozen:
cache_key = (mydep, mydep.unevaluated_atom)
try:
-   return self.xcache[level][cache_key][:]
+   
coroutine_return(self.xcache[level][cache_key][:])
except KeyError:
pass
 
+   loop = asyncio._wrap_loop(loop)
myval =

[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/

2018-09-23 Thread Zac Medico

commit: d9151dfb8144563119f016b439de2b0f53d8858e
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Sep 24 00:20:47 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Sep 24 00:45:40 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d9151dfb

portdbapi.xmatch: remove deprecated *match-list

The bestmatch-list and match-list level arguments have been deprecated
since v2.2.0, commit 0d375f1105ad67c8b7e3b16b57cdbb367f99cd69.

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/dbapi/porttree.py | 12 
 1 file changed, 12 deletions(-)

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index aa8b50a57..56955ec34 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1165,18 +1165,6 @@ class portdbapi(dbapi):
else:
myval = ""
 
-   elif level == "bestmatch-list":
-   #dep match -- find best match but restrict search to 
sublist
-   warnings.warn("The 'bestmatch-list' mode of "
-   "portage.dbapi.porttree.portdbapi.xmatch is 
deprecated",
-   DeprecationWarning, stacklevel=2)
-   myval = best(list(self._iter_match(mydep, mylist)))
-   elif level == "match-list":
-   #dep match -- find all matches but restrict search to 
sublist (used in 2nd half of visible())
-   warnings.warn("The 'match-list' mode of "
-   "portage.dbapi.porttree.portdbapi.xmatch is 
deprecated",
-   DeprecationWarning, stacklevel=2)
-   myval = list(self._iter_match(mydep, mylist))
else:
raise AssertionError(
"Invalid level argument: '%s'" % level)

92 matches

Mail list logo