[gentoo-portage-dev] [PATCH v2] find_smallest_cycle: enhance search prioritization

2020-11-21 Thread Zac Medico
Enhance the find_smallest_cycle function to prioritize its
search so that it will minimize the use of installed packages
to break cycles. When installed packages must be used to
break cycles, it will now prefer to do this for runtime
dependencies over buildtime dependencies, since it's
preferable to build against latest versions of buildtime
dependencies whenever possible. This should solve some cases
of bug 199856 which have been triggered by unsafe reliance
on installed packages to break cycles.

The included unit test case demonstrates correct merge order
for a dependency calculation involving 6 independent cycles.
This test case fails with the master branch, due to a buildtime
dependency cycle of 3 packages being merged earlier than cycles
of 2 packages. We can generalize this to say that the master
branch may use an installed package to break an arbitrarily
sized cycle in a somewhat random location, even though that
cycle may be composed of smaller independent cycles which
would be safer to break individually.

Bug: https://bugs.gentoo.org/754903
Signed-off-by: Zac Medico 
---
[PATCH v2]
* Add a unit test case which demonstrates a significant flaw
  in the master branch.
* Sort nodes in find_smallest_cycle, for deterministic results. 

 lib/_emerge/DepPriorityNormalRange.py |  2 +
 lib/_emerge/DepPrioritySatisfiedRange.py  | 52 ++-
 lib/_emerge/depgraph.py   | 43 +--
 .../tests/resolver/test_merge_order.py| 10 
 4 files changed, 66 insertions(+), 41 deletions(-)

diff --git a/lib/_emerge/DepPriorityNormalRange.py 
b/lib/_emerge/DepPriorityNormalRange.py
index 5f3f3da70..10f205a3b 100644
--- a/lib/_emerge/DepPriorityNormalRange.py
+++ b/lib/_emerge/DepPriorityNormalRange.py
@@ -14,6 +14,7 @@ class DepPriorityNormalRange:
"""
MEDIUM  = 3
MEDIUM_SOFT = 2
+   MEDIUM_POST = 2
SOFT= 1
NONE= 0
 
@@ -37,6 +38,7 @@ class DepPriorityNormalRange:
 
ignore_medium  = _ignore_runtime
ignore_medium_soft = _ignore_runtime_post
+   ignore_medium_post = _ignore_runtime_post
ignore_soft= _ignore_optional
 
 DepPriorityNormalRange.ignore_priority = (
diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py 
b/lib/_emerge/DepPrioritySatisfiedRange.py
index e056e676f..df2439f1f 100644
--- a/lib/_emerge/DepPrioritySatisfiedRange.py
+++ b/lib/_emerge/DepPrioritySatisfiedRange.py
@@ -8,17 +8,18 @@ class DepPrioritySatisfiedRange:
 
not satisfied and buildtimeHARD
not satisfied and runtime  7   MEDIUM
-   not satisfied and runtime_post 6   MEDIUM_SOFT
-   satisfied and buildtime_slot_op5   SOFT
-   satisfied and buildtime4   SOFT
-   satisfied and runtime  3   SOFT
-   satisfied and runtime_post 2   SOFT
+   satisfied and buildtime_slot_op6   MEDIUM_SOFT
+   satisfied and buildtime5   MEDIUM_SOFT
+   satisfied and runtime  4   MEDIUM_SOFT
+   runtime_post   3   MEDIUM_POST
+   satisfied and runtime_post 2   MEDIUM_POST
optional   1   SOFT
(none of the above)0   NONE
"""
MEDIUM  = 7
MEDIUM_SOFT = 6
-   SOFT= 5
+   MEDIUM_POST = 3
+   SOFT= 1
NONE= 0
 
@classmethod
@@ -37,15 +38,23 @@ class DepPrioritySatisfiedRange:
return False
return bool(priority.runtime_post)
 
+   @classmethod
+   def _ignore_runtime_post(cls, priority):
+   if priority.__class__ is not DepPriority:
+   return False
+   return bool(priority.optional or priority.runtime_post)
+
@classmethod
def _ignore_satisfied_runtime(cls, priority):
if priority.__class__ is not DepPriority:
return False
if priority.optional:
return True
-   if not priority.satisfied:
+   if priority.buildtime:
return False
-   return not priority.buildtime
+   if not priority.runtime:
+   return True
+   return bool(priority.satisfied)
 
@classmethod
def _ignore_satisfied_buildtime(cls, priority):
@@ -61,37 +70,32 @@ class DepPrioritySatisfiedRange:
def _ignore_satisfied_buildtime_slot_op(cls, priority):
if priority.__class__ is not DepPriority:
return False
-   return bool(priority.optional or \
-   priority.satisfied)
-
-   @classmethod
-   def _ignore_runtime_post(cls, priority

[gentoo-portage-dev] [PATCH] find_smallest_cycle: enhance search prioritization

2020-11-19 Thread Zac Medico
Enhance the find_smallest_cycle function to prioritize its
search so that it will minimize the use of installed packages
to break cycles. When installed packages must be used to
break cycles, it will now prefer to do this for runtime
dependencies over buildtime dependencies, since it's
preferable to build against latest versions of buildtime
dependencies whenever possible. This should solve some cases
of bug 199856 which have been triggered by unsafe reliance
on installed packages to break cycles.

Bug: https://bugs.gentoo.org/754903
Signed-off-by: Zac Medico 
---
 lib/_emerge/DepPriorityNormalRange.py|  2 +
 lib/_emerge/DepPrioritySatisfiedRange.py | 52 +---
 lib/_emerge/depgraph.py  | 41 +++
 3 files changed, 54 insertions(+), 41 deletions(-)

diff --git a/lib/_emerge/DepPriorityNormalRange.py 
b/lib/_emerge/DepPriorityNormalRange.py
index 5f3f3da70..10f205a3b 100644
--- a/lib/_emerge/DepPriorityNormalRange.py
+++ b/lib/_emerge/DepPriorityNormalRange.py
@@ -14,6 +14,7 @@ class DepPriorityNormalRange:
"""
MEDIUM  = 3
MEDIUM_SOFT = 2
+   MEDIUM_POST = 2
SOFT= 1
NONE= 0
 
@@ -37,6 +38,7 @@ class DepPriorityNormalRange:
 
ignore_medium  = _ignore_runtime
ignore_medium_soft = _ignore_runtime_post
+   ignore_medium_post = _ignore_runtime_post
ignore_soft= _ignore_optional
 
 DepPriorityNormalRange.ignore_priority = (
diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py 
b/lib/_emerge/DepPrioritySatisfiedRange.py
index e056e676f..df2439f1f 100644
--- a/lib/_emerge/DepPrioritySatisfiedRange.py
+++ b/lib/_emerge/DepPrioritySatisfiedRange.py
@@ -8,17 +8,18 @@ class DepPrioritySatisfiedRange:
 
not satisfied and buildtimeHARD
not satisfied and runtime  7   MEDIUM
-   not satisfied and runtime_post 6   MEDIUM_SOFT
-   satisfied and buildtime_slot_op5   SOFT
-   satisfied and buildtime4   SOFT
-   satisfied and runtime  3   SOFT
-   satisfied and runtime_post 2   SOFT
+   satisfied and buildtime_slot_op6   MEDIUM_SOFT
+   satisfied and buildtime5   MEDIUM_SOFT
+   satisfied and runtime  4   MEDIUM_SOFT
+   runtime_post   3   MEDIUM_POST
+   satisfied and runtime_post 2   MEDIUM_POST
optional   1   SOFT
(none of the above)0   NONE
"""
MEDIUM  = 7
MEDIUM_SOFT = 6
-   SOFT= 5
+   MEDIUM_POST = 3
+   SOFT= 1
NONE= 0
 
@classmethod
@@ -37,15 +38,23 @@ class DepPrioritySatisfiedRange:
return False
return bool(priority.runtime_post)
 
+   @classmethod
+   def _ignore_runtime_post(cls, priority):
+   if priority.__class__ is not DepPriority:
+   return False
+   return bool(priority.optional or priority.runtime_post)
+
@classmethod
def _ignore_satisfied_runtime(cls, priority):
if priority.__class__ is not DepPriority:
return False
if priority.optional:
return True
-   if not priority.satisfied:
+   if priority.buildtime:
return False
-   return not priority.buildtime
+   if not priority.runtime:
+   return True
+   return bool(priority.satisfied)
 
@classmethod
def _ignore_satisfied_buildtime(cls, priority):
@@ -61,37 +70,32 @@ class DepPrioritySatisfiedRange:
def _ignore_satisfied_buildtime_slot_op(cls, priority):
if priority.__class__ is not DepPriority:
return False
-   return bool(priority.optional or \
-   priority.satisfied)
-
-   @classmethod
-   def _ignore_runtime_post(cls, priority):
-   if priority.__class__ is not DepPriority:
-   return False
-   return bool(priority.optional or \
-   priority.satisfied or \
-   priority.runtime_post)
+   if priority.optional:
+   return True
+   if priority.satisfied:
+   return True
+   return not priority.buildtime and not priority.runtime
 
@classmethod
def _ignore_runtime(cls, priority):
if priority.__class__ is not DepPriority:
return False
-   return bool(priority.satisfied or \
-   priority.optional or \
-   

[gentoo-portage-dev] [PATCH] emerge: Disable profile deprecation warning inheritance (bug 753497)

2020-11-08 Thread Zac Medico
According to PMS, a deprecated profile warning is not inherited. Since
the current profile node may have been inherited by a user profile
node, the deprecation warning may be relevant even if it is not a
top-level profile node. Therefore, consider the deprecated warning
to be irrelevant when the current profile node belongs to the same
repo as the previous profile node.

Bug: https://bugs.gentoo.org/753497
Signed-off-by: Zac Medico 
---
 .../ebuild/_config/LocationsManager.py| 30 ++-
 .../ebuild/deprecated_profile_check.py|  9 +++---
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/lib/portage/package/ebuild/_config/LocationsManager.py 
b/lib/portage/package/ebuild/_config/LocationsManager.py
index b90b9227c..28740b355 100644
--- a/lib/portage/package/ebuild/_config/LocationsManager.py
+++ b/lib/portage/package/ebuild/_config/LocationsManager.py
@@ -30,7 +30,9 @@ _PORTAGE1_DIRECTORIES = frozenset([
 
 _profile_node = collections.namedtuple('_profile_node',
('location', 'portage1_directories', 'user_config',
-   'profile_formats', 'eapi', 'allow_build_id'))
+   'profile_formats', 'eapi', 'allow_build_id',
+   'show_deprecated_warning',
+))
 
 _allow_parent_colon = frozenset(
["portage-2"])
@@ -132,7 +134,7 @@ class LocationsManager:
if self.profile_path:
try:

self._addProfile(os.path.realpath(self.profile_path),
-   repositories, known_repos)
+   repositories, known_repos, ())
except ParseError as e:
if not portage._sync_mode:
writemsg(_("!!! Unable to parse 
profile: '%s'\n") % self.profile_path, noiselevel=-1)
@@ -154,7 +156,9 @@ class LocationsManager:
('profile-bashrcs', 'profile-set'),
read_corresponding_eapi_file(
custom_prof + os.sep, default=None),
-   True))
+   True,
+   show_deprecated_warning=False,
+   ))
del custom_prof
 
self.profiles = tuple(self.profiles)
@@ -167,7 +171,7 @@ class LocationsManager:
noiselevel=-1)
raise DirectoryNotFound(var)
 
-   def _addProfile(self, currentPath, repositories, known_repos):
+   def _addProfile(self, currentPath, repositories, known_repos, 
previous_repos):
current_abs_path = os.path.abspath(currentPath)
allow_directories = True
allow_parent_colon = True
@@ -176,8 +180,8 @@ class LocationsManager:
current_formats = ()
eapi = None
 
-   intersecting_repos = [x for x in known_repos
-   if current_abs_path.startswith(x[0])]
+   intersecting_repos = tuple(x for x in known_repos
+   if current_abs_path.startswith(x[0]))
if intersecting_repos:
# Handle nested repositories. The longest path
# will be the correct one.
@@ -214,6 +218,14 @@ class LocationsManager:
for x in layout_data['profile-formats'])
current_formats = tuple(layout_data['profile-formats'])
 
+   # According to PMS, a deprecated profile warning is not 
inherited. Since
+   # the current profile node may have been inherited by a user 
profile
+   # node, the deprecation warning may be relevant even if it is 
not a
+   # top-level profile node. Therefore, consider the deprecated 
warning
+   # to be irrelevant when the current profile node belongs to the 
same
+   # repo as the previous profile node.
+   show_deprecated_warning = (not previous_repos or
+   tuple(x[0] for x in previous_repos) != tuple(x[0] for x 
in intersecting_repos))
 
if compat_mode:
offenders = 
_PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath))
@@ -256,7 +268,7 @@ class LocationsManager:
parentPath = 
os.path.realpath(parentPath)
 
if exists_raise_eaccess(parentPath):
-   self._addProfile(parentPath, 
repositories, known_repos)
+   self._addProfile(parentPath, 
repositories, known_repos, intersecting_repos)
else:
raise ParseError(
_("Parent '%s' not found: 
'%s'") 

[gentoo-portage-dev] [PATCH] make.conf: expand special *ROOT variables (bug 752147)

2020-11-01 Thread Zac Medico
Bug: https://bugs.gentoo.org/752147
Signed-off-by: Zac Medico 
---
 lib/portage/package/ebuild/config.py | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/portage/package/ebuild/config.py 
b/lib/portage/package/ebuild/config.py
index a09fdbced..3be8f9f6d 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -401,9 +401,14 @@ class config:
expand_map = env_d.copy()
self._expand_map = expand_map
 
-   # Allow make.globals to set default paths relative to 
${EPREFIX}.
+   # Allow make.globals and make.conf to set default paths 
relative to vars like ${EPREFIX}.
+   expand_map["BROOT"] = broot
expand_map["EPREFIX"] = eprefix
+   expand_map["EROOT"] = eroot
+   expand_map["ESYSROOT"] = esysroot
expand_map["PORTAGE_CONFIGROOT"] = config_root
+   expand_map["ROOT"] = target_root
+   expand_map["SYSROOT"] = sysroot
 
if portage._not_installed:
make_globals_path = 
os.path.join(PORTAGE_BASE_PATH, "cnf", "make.globals")
-- 
2.26.2




[gentoo-portage-dev] [PATCH] emerge: add --quickpkg-direct-root option

2020-11-01 Thread Zac Medico
Specify the root to use as the --quickpkg-direct package source. This
root is assumed to be immutable during the entire emerge operation.
The default is set to "/".

Bug: https://bugs.gentoo.org/752066
Signed-off-by: Zac Medico 
---
 lib/_emerge/actions.py  | 19 ---
 lib/_emerge/depgraph.py | 11 +--
 lib/_emerge/main.py |  5 +
 lib/portage/tests/emerge/test_simple.py |  3 ++-
 man/emerge.1| 10 --
 5 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 5e8a46957..2e155899c 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -49,7 +49,7 @@ from portage.package.ebuild._ipc.QueryCommand import 
QueryCommand
 from portage.package.ebuild.fetch import _hide_url_passwd
 from portage._sets import load_default_config, SETPREFIX
 from portage._sets.base import InternalPackageSet
-from portage.util import cmp_sort_key, writemsg, varexpand, \
+from portage.util import cmp_sort_key, normalize_path, writemsg, varexpand, \
writemsg_level, writemsg_stdout
 from portage.util.digraph import digraph
 from portage.util.SlotObject import SlotObject
@@ -106,13 +106,26 @@ def action_build(emerge_config, trees=DeprecationWarning,
# before we get here, so warn if they're not (bug #267103).
chk_updated_cfg_files(settings['EROOT'], ['/etc/portage'])
 
+   quickpkg_root = normalize_path(os.path.abspath(
+   emerge_config.opts.get('--quickpkg-direct-root',
+   
emerge_config.running_config.settings['ROOT']))).rstrip(os.path.sep) + 
os.path.sep
quickpkg_direct = ("--usepkg" in emerge_config.opts and
emerge_config.opts.get('--quickpkg-direct', 'n') == 'y' and
-   emerge_config.target_config is not emerge_config.running_config)
+   emerge_config.target_config.settings['ROOT'] != quickpkg_root)
if '--getbinpkg' in emerge_config.opts or quickpkg_direct:
kwargs = {}
if quickpkg_direct:
-   kwargs['add_repos'] = 
(emerge_config.running_config.trees['vartree'].dbapi,)
+   if quickpkg_root == emerge_config.running_config.root:
+   quickpkg_vardb = 
emerge_config.running_config.trees['vartree'].dbapi
+   else:
+   quickpkg_settings = portage.config(
+   
config_root=emerge_config.target_config.settings['PORTAGE_CONFIGROOT'],
+   target_root=quickpkg_root,
+   
env=emerge_config.target_config.settings.backupenv,
+   
sysroot=emerge_config.target_config.settings['SYSROOT'],
+   
eprefix=emerge_config.target_config.settings['EPREFIX'])
+   quickpkg_vardb = 
portage.vartree(settings=quickpkg_settings).dbapi
+   kwargs['add_repos'] = (quickpkg_vardb,)
 
try:
emerge_config.target_config.trees['bintree'].populate(
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 0bb0352e7..898cf6c1a 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -41,7 +41,7 @@ from portage._sets import SETPREFIX
 from portage._sets.base import InternalPackageSet
 from portage.util import ConfigProtect, shlex_split, new_protect_filename
 from portage.util import cmp_sort_key, writemsg, writemsg_stdout
-from portage.util import ensure_dirs
+from portage.util import ensure_dirs, normalize_path
 from portage.util import writemsg_level, write_atomic
 from portage.util.digraph import digraph
 from portage.util.futures import asyncio
@@ -4567,8 +4567,15 @@ class depgraph:
self._dynamic_config._skip_restart = True
return False, myfavorites
 
+   # Since --quickpkg-direct assumes that --quickpkg-direct-root is
+   # immutable, assert that there are no merged or unmerge tasks
+   # for --quickpkg-direct-root.
+   quickpkg_root = normalize_path(os.path.abspath(
+   self._frozen_config.myopts.get('--quickpkg-direct-root',
+   
self._frozen_config._running_root.settings['ROOT']))).rstrip(os.path.sep) + 
os.path.sep
if (self._frozen_config.myopts.get('--quickpkg-direct', 'n') == 
'y' and
-   self._frozen_config.target_root is not 
self._frozen_config._running_root):
+   self._frozen_config.settings['ROOT'] != quickpkg_root 
and
+   self._frozen_config._running_root.settings['ROOT'] == 
quickpkg_root):
running_root = self._frozen_config._running_root.root

[gentoo-portage-dev] [PATCH] pid-sandbox: Forward SIGTSTP and SIGCONT (bug 704498)

2020-10-28 Thread Zac Medico
For correct operation of Ctrl+Z, forward SIGTSTP and SIGCONT
to all sandboxed pids.

Fixes: 37e4dc5ae842 ("pid-sandbox: pid-ns-init setsid support (bug 675870)")
Bug: https://bugs.gentoo.org/704498
Signed-off-by: Zac Medico 
---
 bin/pid-ns-init | 24 +++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/bin/pid-ns-init b/bin/pid-ns-init
index 3a218a5df..e410dd028 100644
--- a/bin/pid-ns-init
+++ b/bin/pid-ns-init
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright 2018-2019 Gentoo Authors
+# Copyright 2018-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -19,6 +19,11 @@ KILL_SIGNALS = (
signal.SIGHUP,
 )
 
+SIGTSTP_SIGCONT = (
+   signal.SIGTSTP,
+   signal.SIGCONT,
+)
+
 
 def forward_kill_signal(pid, signum, frame):
if pid == 0:
@@ -28,6 +33,18 @@ def forward_kill_signal(pid, signum, frame):
os.kill(pid, signum)
 
 
+def forward_sigtstp_sigcont(pid, signum, frame):
+   handler = None
+   if pid == 0:
+   # Temporarily disable the handler in order to prevent it from
+   # being called recursively, since the signal will also be sent
+   # to the current process.
+   handler = signal.signal(signum, signal.SIG_DFL)
+   os.kill(pid, signum)
+   if handler is not None:
+   signal.signal(signum, handler)
+
+
 def preexec_fn(uid, gid, groups, umask):
if gid is not None:
os.setgid(gid)
@@ -97,6 +114,11 @@ def main(argv):
for signum in KILL_SIGNALS:
signal.signal(signum, sig_handler)
 
+   # For correct operation of Ctrl+Z, forward SIGTSTP and SIGCONT.
+   sigtstp_sigcont_handler = functools.partial(forward_sigtstp_sigcont, 0 
if setsid else main_child_pid)
+   for signum in SIGTSTP_SIGCONT:
+   signal.signal(signum, sigtstp_sigcont_handler)
+
# wait for child processes
while True:
try:
-- 
2.26.2




[gentoo-portage-dev] [PATCH v2] emerge: enable parallel-fetch during pkg_pretend (bug 710432)

2020-09-20 Thread Zac Medico
Execute pkg_pretend phases in a coroutine while parallel-fetch
is running concurrently. When it's time to execute the pkg_pretend
phase for a remote binary package, use a Scheduler _get_prefetcher
method to get a running prefetcher if available, and otherwise
start a new fetcher.

Since pkg_pretend phases now run inside of the --keep-going retry
loop, --keep-going is now able to recover from pkg_pretend
failures, which fixes bug 404157.

Bug: https://bugs.gentoo.org/404157
Bug: https://bugs.gentoo.org/710432
Signed-off-by: Zac Medico 
---
[PATCH v2] records failed packages for correct interaction with
emerge --keep-going, which fixes bug 404157

 lib/_emerge/Scheduler.py | 142 +++
 1 file changed, 99 insertions(+), 43 deletions(-)

diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index a69421288..465f928a0 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -25,6 +25,7 @@ from portage._sets import SETPREFIX
 from portage._sets.base import InternalPackageSet
 from portage.util import ensure_dirs, writemsg, writemsg_level
 from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
 from portage.util.SlotObject import SlotObject
 from portage.util._async.SchedulerInterface import SchedulerInterface
 from portage.package.ebuild.digestcheck import digestcheck
@@ -766,7 +767,8 @@ class Scheduler(PollScheduler):
 
return prefetcher
 
-   def _run_pkg_pretend(self):
+   @coroutine
+   def _run_pkg_pretend(self, loop=None):
"""
Since pkg_pretend output may be important, this method sends all
output directly to stdout (regardless of options like --quiet or
@@ -774,7 +776,7 @@ class Scheduler(PollScheduler):
"""
 
failures = 0
-   sched_iface = self._sched_iface
+   sched_iface = loop = asyncio._wrap_loop(loop or 
self._sched_iface)
 
for x in self._mergelist:
if not isinstance(x, Package):
@@ -789,18 +791,28 @@ class Scheduler(PollScheduler):
if "pretend" not in x.defined_phases:
continue
 
-   out_str =">>> Running pre-merge checks for " + 
colorize("INFORM", x.cpv) + "\n"
-   portage.util.writemsg_stdout(out_str, noiselevel=-1)
+   out_str = "Running pre-merge checks for " + 
colorize("INFORM", x.cpv)
+   self._status_msg(out_str)
 
root_config = x.root_config
-   settings = self.pkgsettings[root_config.root]
+   settings = self._allocate_config(root_config.root)
settings.setcpv(x)
+   if not x.built:
+   # Get required SRC_URI metadata (it's not 
cached in x.metadata
+   # because some packages have an extremely large 
SRC_URI value).
+   portdb = root_config.trees["porttree"].dbapi
+   (settings.configdict["pkg"]["SRC_URI"],) = 
yield portdb.async_aux_get(
+   x.cpv, ["SRC_URI"], myrepo=x.repo, 
loop=loop
+   )
 
# setcpv/package.env allows for per-package 
PORTAGE_TMPDIR so we
# have to validate it for each package
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
-   return rval
+   failures += 1
+   self._record_pkg_failure(x, settings, FAILURE)
+   self._deallocate_config(settings)
+   continue
 
build_dir_path = os.path.join(
os.path.realpath(settings["PORTAGE_TMPDIR"]),
@@ -809,7 +821,7 @@ class Scheduler(PollScheduler):
settings["PORTAGE_BUILDDIR"] = build_dir_path
build_dir = EbuildBuildDir(scheduler=sched_iface,
settings=settings)
-   sched_iface.run_until_complete(build_dir.async_lock())
+   yield build_dir.async_lock()
current_task = None
 
try:
@@ -835,7 +847,7 @@ class Scheduler(PollScheduler):
phase='clean', 
scheduler=sched_iface, settings=settings)
current_task = clean_phase
clean_phase.start()
-  

[gentoo-portage-dev] [PATCH] emerge: enable parallel-fetch during pkg_pretend (bug 710432)

2020-09-19 Thread Zac Medico
Execute pkg_pretend phases in a coroutine while parallel-fetch
is running concurrently. When it's time to execute the pkg_pretend
phase for a remote binary package, use a Scheduler _get_prefetcher
method to get a running prefetcher if available, and otherwise
start a new fetcher.

Bug: https://bugs.gentoo.org/710432
Signed-off-by: Zac Medico 
---
 lib/_emerge/Scheduler.py | 94 +---
 1 file changed, 58 insertions(+), 36 deletions(-)

diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index a69421288..20884986f 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -25,6 +25,7 @@ from portage._sets import SETPREFIX
 from portage._sets.base import InternalPackageSet
 from portage.util import ensure_dirs, writemsg, writemsg_level
 from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
 from portage.util.SlotObject import SlotObject
 from portage.util._async.SchedulerInterface import SchedulerInterface
 from portage.package.ebuild.digestcheck import digestcheck
@@ -766,7 +767,8 @@ class Scheduler(PollScheduler):
 
return prefetcher
 
-   def _run_pkg_pretend(self):
+   @coroutine
+   def _run_pkg_pretend(self, loop=None):
"""
Since pkg_pretend output may be important, this method sends all
output directly to stdout (regardless of options like --quiet or
@@ -774,7 +776,7 @@ class Scheduler(PollScheduler):
"""
 
failures = 0
-   sched_iface = self._sched_iface
+   sched_iface = loop = asyncio._wrap_loop(loop or 
self._sched_iface)
 
for x in self._mergelist:
if not isinstance(x, Package):
@@ -795,12 +797,18 @@ class Scheduler(PollScheduler):
root_config = x.root_config
settings = self.pkgsettings[root_config.root]
settings.setcpv(x)
+   if not x.built:
+   # Get required SRC_URI metadata (it's not 
cached in x.metadata
+   # because some packages have an extremely large 
SRC_URI value).
+   portdb = root_config.trees["porttree"].dbapi
+   settings.configdict["pkg"]["SRC_URI"], = (yield 
portdb.async_aux_get(
+   x.cpv, ["SRC_URI"], myrepo=x.repo, 
loop=loop))
 
# setcpv/package.env allows for per-package 
PORTAGE_TMPDIR so we
# have to validate it for each package
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
-   return rval
+   coroutine_return(rval)
 
build_dir_path = os.path.join(
os.path.realpath(settings["PORTAGE_TMPDIR"]),
@@ -809,7 +817,7 @@ class Scheduler(PollScheduler):
settings["PORTAGE_BUILDDIR"] = build_dir_path
build_dir = EbuildBuildDir(scheduler=sched_iface,
settings=settings)
-   sched_iface.run_until_complete(build_dir.async_lock())
+   yield build_dir.async_lock()
current_task = None
 
try:
@@ -835,7 +843,7 @@ class Scheduler(PollScheduler):
phase='clean', 
scheduler=sched_iface, settings=settings)
current_task = clean_phase
clean_phase.start()
-   clean_phase.wait()
+   yield clean_phase.async_wait()
 
if x.built:
tree = "bintree"
@@ -845,10 +853,11 @@ class Scheduler(PollScheduler):
# Display fetch on stdout, so that it's 
always clear what
# is consuming time here.
if bintree.isremote(x.cpv):
-   fetcher = BinpkgFetcher(pkg=x,
-   scheduler=sched_iface)
-   fetcher.start()
-   if fetcher.wait() != os.EX_OK:
+   fetcher = 
self._get_prefetcher(x)
+   if fetcher is None:
+   fetcher = 
BinpkgFetcher(pkg=x, scheduler=loop)
+   

[gentoo-portage-dev] [PATCH] _slot_confict_backtrack: group similar missed updates (bug 743115)

2020-09-19 Thread Zac Medico
When a slot conflict occurs due to a missed update, and some other
similar update(s) are available, add the similar update(s) to the
runtime package mask for the same backtracking choice. This reduces
minimum number of backtrack tries required to solve the test case
for bug 743115 from 7 to 4, where the difference of 3 corresponds
to the number of other similar setuptools updates available.

Bug: https://bugs.gentoo.org/743115
Signed-off-by: Zac Medico 
---
 lib/_emerge/depgraph.py   | 25 ---
 lib/_emerge/resolver/backtracking.py  |  7 +++---
 .../test_slot_operator_missed_update.py   |  2 +-
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 7281d8692..2a840b2ca 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1795,15 +1795,32 @@ class depgraph:

self._dynamic_config._parent_atoms.get(to_be_masked, set())
conflict_atoms = set(parent_atom for parent_atom in 
all_parents \
if parent_atom not in parent_atoms)
-   backtrack_data.append((to_be_masked, conflict_atoms))
+
+   similar_pkgs = []
+   if conflict_atoms:
+   # If the conflict has been triggered by a 
missed update, then
+   # we can avoid excessive backtracking if we 
detect similar missed
+   # updates and mask them as part of the same 
backtracking choice.
+   for similar_pkg in 
self._iter_similar_available(to_be_masked, slot_atom):
+   if similar_pkg in conflict_pkgs:
+   continue
+   similar_conflict_atoms = []
+   for parent_atom in conflict_atoms:
+   parent, atom = parent_atom
+   if not atom.match(similar_pkg):
+   
similar_conflict_atoms.append(parent_atom)
+   if similar_conflict_atoms:
+   
similar_pkgs.append((similar_pkg, set(similar_conflict_atoms)))
+   similar_pkgs.append((to_be_masked, conflict_atoms))
+   backtrack_data.append(tuple(similar_pkgs))
 
# Prefer choices that minimize conflict atoms. This is intended
# to take precedence over the earlier package version sort. The
# package version sort is still needed or else choices for the
# testOverlapSlotConflict method of 
VirtualMinimizeChildrenTestCase
# become non-deterministic.
-   backtrack_data.sort(key=lambda item: len(item[1]))
-   to_be_masked = backtrack_data[-1][0]
+   backtrack_data.sort(key=lambda similar_pkgs: max(len(item[1]) 
for item in similar_pkgs))
+   to_be_masked = [item[0] for item in backtrack_data[-1]]
 
self._dynamic_config._backtrack_infos.setdefault(
"slot conflict", []).append(backtrack_data)
@@ -1814,7 +1831,7 @@ class depgraph:
"",
"backtracking due to slot conflict:",
"   first package:  %s" % existing_node,
-   "  package to mask: %s" % to_be_masked,
+   "  package(s) to mask: %s" % str(to_be_masked),
"  slot: %s" % slot_atom,
"   parents: %s" % ", ".join(
"(%s, '%s')" % (ppkg, atom) for ppkg, 
atom in all_parents
diff --git a/lib/_emerge/resolver/backtracking.py 
b/lib/_emerge/resolver/backtracking.py
index bc3fb3206..ca94623ac 100644
--- a/lib/_emerge/resolver/backtracking.py
+++ b/lib/_emerge/resolver/backtracking.py
@@ -166,13 +166,14 @@ class Backtracker:
self._feedback_slot_conflict(conflicts_data[0])
 
def _feedback_slot_conflict(self, conflict_data):
-   for pkg, parent_atoms in conflict_data:
+   for similar_pkgs in conflict_data:
new_node = copy.deepcopy(self._current_node)
new_node.depth += 1
new_node.mask_steps += 1
new_node.terminal = False
-   new_node.parameter.runtime_pkg_mask.setdefault(
-   pkg, {})["slot conflict"] = parent_atoms
+   for pkg, parent_atoms in similar_pkgs:
+   

[gentoo-portage-dev] Re: [PATCH] _slot_confict_backtrack: minimize conflict atoms (bug 743631)

2020-09-19 Thread Zac Medico
On 9/19/20 2:40 PM, Zac Medico wrote:
> diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
> index 3f864aefc..40e7d1325 100644
> --- a/lib/_emerge/depgraph.py
> +++ b/lib/_emerge/depgraph.py
> @@ -1797,6 +1797,12 @@ class depgraph:
>   if parent_atom not in parent_atoms)
>   backtrack_data.append((to_be_masked, conflict_atoms))
>  
> + # Prefer choices that minimize conflict atoms. This is intended
> + # to take precedence over the earlier package version sort. The
> + # package version sort is still needed or else the
> + # testOverlapSlotConflict method of 
> VirtualMinimizeChildrenTestCase
> + # will not succeed reliably with the default backtrack limit.

Updated this comment to indicate that the package version sort sort is actually
needed for deterministic results:

# Prefer choices that minimize conflict atoms. This is intended
# to take precedence over the earlier package version sort. The
# package version sort is still needed or else choices for the
# testOverlapSlotConflict method of 
VirtualMinimizeChildrenTestCase
# become non-deterministic.

-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH] _slot_confict_backtrack: minimize conflict atoms (bug 743631)

2020-09-19 Thread Zac Medico
Prefer choices that minimize conflict atoms, so that choices
which satisfy all parents are preferred. This reduces the
minimum necessary backtrack tries from 21 to 7 for the unit
test related to bug 743115.

Bug: https://bugs.gentoo.org/743115
Bug: https://bugs.gentoo.org/743631
Signed-off-by: Zac Medico 
---
 lib/_emerge/depgraph.py | 6 ++
 .../tests/resolver/test_slot_operator_missed_update.py  | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 3f864aefc..40e7d1325 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1797,6 +1797,12 @@ class depgraph:
if parent_atom not in parent_atoms)
backtrack_data.append((to_be_masked, conflict_atoms))
 
+   # Prefer choices that minimize conflict atoms. This is intended
+   # to take precedence over the earlier package version sort. The
+   # package version sort is still needed or else the
+   # testOverlapSlotConflict method of 
VirtualMinimizeChildrenTestCase
+   # will not succeed reliably with the default backtrack limit.
+   backtrack_data.sort(key=lambda item: len(item[1]))
to_be_masked = backtrack_data[-1][0]
 
self._dynamic_config._backtrack_infos.setdefault(
diff --git a/lib/portage/tests/resolver/test_slot_operator_missed_update.py 
b/lib/portage/tests/resolver/test_slot_operator_missed_update.py
index fce012f62..1ea701003 100644
--- a/lib/portage/tests/resolver/test_slot_operator_missed_update.py
+++ b/lib/portage/tests/resolver/test_slot_operator_missed_update.py
@@ -90,7 +90,7 @@ class BacktrackMissedUpdateTestCase(TestCase):
# Bug 743115: missed updates trigger excessive 
backtracking
ResolverPlaygroundTestCase(
[">=dev-python/pypy3-7.3.2_rc", "@world"],
-   options={"--update": True, "--deep": True, 
"--backtrack": 25},
+   options={"--update": True, "--deep": True, 
"--backtrack": 10},
success=True,
mergelist=[
"dev-python/pypy3-7.3.2_rc2_p37-r1",
-- 
2.25.3




[gentoo-portage-dev] [PATCH] egencache: add --external-cache-only option (bug 741474)

2020-09-12 Thread Zac Medico
The --external-cache-only option is useful for client-side
use cases where writing cache files inside the repository itself
may interfere with repository verification. This option is
currently supported for --update and --update-pkg-desc-index
actions, for which consumers of the corresponding cache or
index files are already capable of consuming files from the
external cache directory (/var/cache/edb/dep).

Bug: https://bugs.gentoo.org/737470
Signed-off-by: Zac Medico 
---
 bin/egencache   | 44 ++---
 cnf/repo.postsync.d/example | 19 
 man/egencache.1 |  7 ++
 3 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/bin/egencache b/bin/egencache
index 532e37f20..4ee63edad 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -107,6 +107,9 @@ def parse_args(args):
common.add_argument("--config-root",
help="location of portage config files",
dest="portage_configroot")
+   common.add_argument("--external-cache-only",
+   action="store_true",
+   help="Output only to the external cache (not the repository 
itself)")
common.add_argument("--gpg-dir",
help="override the PORTAGE_GPG_DIR variable",
dest="gpg_dir")
@@ -246,7 +249,7 @@ def parse_args(args):
 
 class GenCache:
def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None,
-   rsync=False):
+   rsync=False, external_cache_only=False):
# The caller must set portdb.porttrees in order to constrain
# findname, cp_list, and cpv_list to the desired tree.
tree = portdb.porttrees[0]
@@ -263,18 +266,21 @@ class GenCache:
else:
self._cp_set = None
self._cp_missing = set()
-   write_auxdb = "metadata-transfer" in portdb.settings.features
+   write_auxdb = external_cache_only or "metadata-transfer" in 
portdb.settings.features
self._regen = MetadataRegen(portdb, cp_iter=cp_iter,
consumer=self._metadata_callback,
max_jobs=max_jobs, max_load=max_load,
write_auxdb=write_auxdb, main=True)
self.returncode = os.EX_OK
conf = portdb.repositories.get_repo_for_location(tree)
-   self._trg_caches = tuple(conf.iter_pregenerated_caches(
-   self._auxdbkeys, force=True, readonly=False))
-   if not self._trg_caches:
-   raise Exception("cache formats '%s' aren't supported" %
-   (" ".join(conf.cache_formats),))
+   if external_cache_only:
+   self._trg_caches = ()
+   else:
+   self._trg_caches = tuple(conf.iter_pregenerated_caches(
+   self._auxdbkeys, force=True, readonly=False))
+   if not self._trg_caches:
+   raise Exception("cache formats '%s' aren't 
supported" %
+   (" ".join(conf.cache_formats),))
 
if rsync:
for trg_cache in self._trg_caches:
@@ -1092,7 +1098,8 @@ def egencache_main(args):
gen_cache = GenCache(portdb, cp_iter=cp_iter,
max_jobs=options.jobs,
max_load=options.load_average,
-   rsync=options.rsync)
+   rsync=options.rsync,
+   external_cache_only=options.external_cache_only)
gen_cache.run()
if options.tolerant:
ret.append(os.EX_OK)
@@ -1100,20 +1107,21 @@ def egencache_main(args):
ret.append(gen_cache.returncode)
 
if options.update_pkg_desc_index:
-   if repo_config.writable:
+   if not options.external_cache_only and repo_config.writable:
writable_location = repo_config.location
else:
writable_location = os.path.join(portdb.depcachedir,
repo_config.location.lstrip(os.sep))
-   msg = [
-   "WARNING: Repository is not writable: %s" % (
-   repo_config.location,),
-   " Using cache directory instead: %s" % (
-   writable_location,)
-   ]
-   msg = "".join(line + '\n' for line in msg)
-   writemsg_level(msg,
-   level=logging.WARNING, noiselevel=-1)
+

[gentoo-portage-dev] [PATCH] get_mirror_url: urlquote only for ftp, http, and https (bug 741474)

2020-09-12 Thread Zac Medico
It's necessary to use urlquote for correct behavior with ftp, http,
and https, since it's possible for file names to contain percent
encoded characters that need to be protected by an additional layer
of percent encoding. For other protocols such as sftp and rsync,
all characters are interpreted literally, so urlquote must not be
used.

Fixes: c238d5f7ed264179c263f5a2da983c4ee50b4f00
Bug: https://bugs.gentoo.org/719810
Bug: https://bugs.gentoo.org/741474
Signed-off-by: Zac Medico 
---
 lib/portage/package/ebuild/fetch.py | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/portage/package/ebuild/fetch.py 
b/lib/portage/package/ebuild/fetch.py
index 7c61fe463..ca031f31e 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -513,8 +513,12 @@ def get_mirror_url(mirror_url, filename, mysettings, 
cache_path=None):
json.dump(cache, f)
f.close()
 
-   return (mirror_url + "/distfiles/" +
-   
urlquote(mirror_conf.get_best_supported_layout().get_path(filename)))
+   # For some protocols, urlquote is required for correct behavior,
+   # and it must not be used for other protocols like rsync and sftp.
+   path = mirror_conf.get_best_supported_layout().get_path(filename)
+   if urlparse(mirror_url).scheme in ('ftp', 'http', 'https'):
+   path = urlquote(path)
+   return mirror_url + "/distfiles/" + path
 
 
 def fetch(myuris, mysettings, listonly=0, fetchonly=0,
-- 
2.25.3




[gentoo-portage-dev] [PATCH] selinux: encode os path arguments as UTF-8 (bug 741194)

2020-09-08 Thread Zac Medico
Encode path arguments as UTF-8, like portage.os wrapper.

Fixes: 6137290b2bb8 ("selinux: python3 unicode paths, bug #430488")
Bug: https://bugs.gentoo.org/741194
Signed-off-by: Zac Medico 
---
 lib/portage/_selinux.py | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/portage/_selinux.py b/lib/portage/_selinux.py
index e3e18c0b8..2423ff8d6 100644
--- a/lib/portage/_selinux.py
+++ b/lib/portage/_selinux.py
@@ -14,7 +14,7 @@ except ImportError:
 
 import portage
 from portage import _encodings
-from portage import _native_string
+from portage import _native_string, _unicode_encode
 from portage.localization import _
 
 def copyfile(src, dest):
@@ -41,7 +41,6 @@ def is_selinux_enabled():
return selinux.is_selinux_enabled()
 
 def mkdir(target, refdir):
-   target = _native_string(target, encoding=_encodings['fs'], 
errors='strict')
refdir = _native_string(refdir, encoding=_encodings['fs'], 
errors='strict')
(rc, ctx) = selinux.getfilecon(refdir)
if rc < 0:
@@ -51,20 +50,21 @@ def mkdir(target, refdir):
 
setfscreate(ctx)
try:
-   os.mkdir(target)
+   os.mkdir(_unicode_encode(target, encoding=_encodings['fs'], 
errors='strict'))
finally:
setfscreate()
 
 def rename(src, dest):
src = _native_string(src, encoding=_encodings['fs'], errors='strict')
-   dest = _native_string(dest, encoding=_encodings['fs'], errors='strict')
(rc, ctx) = selinux.lgetfilecon(src)
if rc < 0:
raise OSError(_("rename: Failed getting context of \"%s\".") % 
src)
 
setfscreate(ctx)
try:
-   os.rename(src, dest)
+   os.rename(
+   _unicode_encode(src, encoding=_encodings['fs'], 
errors='strict'),
+   _unicode_encode(dest, encoding=_encodings['fs'], 
errors='strict'))
finally:
setfscreate()
 
@@ -132,8 +132,6 @@ class spawn_wrapper:
return self._spawn_func(*args, **kwargs)
 
 def symlink(target, link, reflnk):
-   target = _native_string(target, encoding=_encodings['fs'], 
errors='strict')
-   link = _native_string(link, encoding=_encodings['fs'], errors='strict')
reflnk = _native_string(reflnk, encoding=_encodings['fs'], 
errors='strict')
(rc, ctx) = selinux.lgetfilecon(reflnk)
if rc < 0:
@@ -143,6 +141,8 @@ def symlink(target, link, reflnk):
 
setfscreate(ctx)
try:
-   os.symlink(target, link)
+   os.symlink(
+   _unicode_encode(target, encoding=_encodings['fs'], 
errors='strict'),
+   _unicode_encode(link, encoding=_encodings['fs'], 
errors='strict'))
finally:
setfscreate()
-- 
2.25.3




Re: [gentoo-portage-dev] [PATCH v5] env-update: create systemd user-session environment definition

2020-09-07 Thread Zac Medico
On 9/5/20 12:18 AM, Florian Schmaus wrote:
> Portage's env-update currently transforms the environment information
> from /etc/env.d into /etc/profile.env, which is typically sourced by
> every user session, setting up its environment.
> 
> However, /etc/profile.env is not sourced by systemd user
> services. Instead, for the definition of a systemd user session
> environment, the 'environment.d' machinery exists. Unfortunately, up
> to now, env-update does not produce a profile.env equivalent for this
> machinery, causing issues for systemd user services. For example, an
> emacs daemon run as user systemd service does not have a complete
> PATH (bug #704412 [1]), because some PATH components are injected by
> packages via /etc/env.d. For example, an LLVM ebuild may set
> PATH="/usr/lib/llvm/9/bin".
> 
> This commit changes env-update so that a systemd user session
> environment configuration file named
> 
> /etc/environment.d/10-gentoo-env.conf
> 
> is created.
> 
> Thanks to Michael 'veremitz' Everitt, Arfrever Frehtes Taifersar
> Arahesis, Ulrich Müller, Joakim Tjernlund, and Zac Medico for the
> useful feedback.
> 
> 1: https://bugs.gentoo.org/704412
> 
> Closes: https://bugs.gentoo.org/704416
> Signed-off-by: Florian Schmaus 
> ---
> 
> Notes:
> - Shorten created filename to 10-gentoo-env.conf
> - Minor fixes in the commit message
> - Use atomic_ofstream()
> - Use os.makedirs() (Thanks Zac)
> 
>  lib/portage/util/env_update.py | 42 +++---
>  1 file changed, 39 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py
> index f130b6f6b..ab3caee47 100644
> --- a/lib/portage/util/env_update.py
> +++ b/lib/portage/util/env_update.py
> @@ -333,14 +333,16 @@ def _env_update(makelinks, target_root, prev_mtimes, 
> contents, env,
>  
>   del specials["LDPATH"]
>  
> - penvnotice  = "# THIS FILE IS AUTOMATICALLY GENERATED BY env-update.\n"
> - penvnotice += "# DO NOT EDIT THIS FILE. CHANGES TO STARTUP PROFILES\n"
> + notice  = "# THIS FILE IS AUTOMATICALLY GENERATED BY env-update.\n"
> + notice += "# DO NOT EDIT THIS FILE."
> + penvnotice  = notice + " CHANGES TO STARTUP PROFILES\n"
>   cenvnotice  = penvnotice[:]
>   penvnotice += "# GO INTO /etc/profile NOT /etc/profile.env\n\n"
>   cenvnotice += "# GO INTO /etc/csh.cshrc NOT /etc/csh.env\n\n"
>  
>   #create /etc/profile.env for bash support
> - outfile = atomic_ofstream(os.path.join(eroot, "etc", "profile.env"))
> + profile_env_path = os.path.join(eroot, "etc", "profile.env")
> + outfile = atomic_ofstream(profile_env_path)
>   outfile.write(penvnotice)
>  
>   env_keys = [x for x in env if x != "LDPATH"]
> @@ -353,6 +355,40 @@ def _env_update(makelinks, target_root, prev_mtimes, 
> contents, env,
>   outfile.write("export %s='%s'\n" % (k, v))
>   outfile.close()
>  
> + # Create the systemd user environment configuration file
> + # /etc/environment.d/10-gentoo-env.conf with the
> + # environment configuration from /etc/env.d.
> + systemd_environment_dir = os.path.join(eroot, "etc", "environment.d")
> + os.makedirs(systemd_environment_dir, exist_ok=True)
> +
> + systemd_gentoo_env_path = os.path.join(systemd_environment_dir,
> +  "10-gentoo-env.conf")
> + systemd_gentoo_env = atomic_ofstream(systemd_gentoo_env_path)
> + try:
> + senvnotice = notice + "\n\n"
> + systemd_gentoo_env.write(senvnotice)
> +
> + for env_key in env_keys:
> + env_key_value = env[env_key]
> +
> + # Skip variables with the empty string
> + # as value. Those sometimes appear in
> + # profile.env (e.g. "export GCC_SPECS=''"),
> + # but are invalid in systemd's syntax.
> + if not env_key_value:
> + continue
> +
> + # Transform into systemd environment.d
> + # conf syntax, basically shell variable
> + # assignment (without "export ").
> + line = f"{env_key}={env_key_value}\n"
> +
> + systemd_gentoo_env.write(line)
> + except:
> + systemd_gentoo_env.abort()
> + raise
> + systemd_gentoo_env.close()
> +
>   #create /etc/csh.env for (t)csh support
>   outfile = atomic_ofstream(os.path.join(eroot, "etc", "csh.env"))
>   outfile.write(cenvnotice)
> 

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=45a5982fe8076066323e91f6b5fe860f3a429f9f
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] Re: [PATCH 0/2] Add binrepos.conf to support fetchcommand customization (bug 661332)

2020-09-07 Thread Zac Medico
On 9/6/20 11:31 PM, Zac Medico wrote:
> Support /etc/portage/binrepos.conf as a replacement for the
> PORTAGE_BINHOST variable. Behavior is similar to repos.conf,
> initially supporting just the sync-uri attribute. Both binrepos.conf
> and PORTAGE_BINHOST can be used simultaneously, in the same way that
> repos.conf and PORTDIR_OVERLAY can be used simultaneously.
> 
> The emerge --info output for binrepos.conf looks like this:
> 
> Binary Repositories:
> 
> example-binhost
>   sync-uri: https://example.com/packages
> 
> Support customization of fetchcommand and resumecommand in
> binrepos.conf, allowing customized authentication mechanisms for
> each repository.
> 
> Zac Medico (2):
>   Add binrepos.conf to replace PORTAGE_BINHOST (bug 668334)
>   binrepos.conf: support fetchcommand customization (bug 661332)
> 
>  lib/_emerge/BinpkgFetcher.py|  29 --
>  lib/_emerge/actions.py  |  13 ++-
>  lib/portage/binrepo/__init__.py |   0
>  lib/portage/binrepo/config.py   | 131 
>  lib/portage/const.py|   1 +
>  lib/portage/dbapi/bintree.py|  48 ++---
>  lib/portage/tests/emerge/test_simple.py |  14 ++-
>  man/make.conf.5 |   3 +-
>  man/portage.5   |  52 ++
>  9 files changed, 264 insertions(+), 27 deletions(-)
>  create mode 100644 lib/portage/binrepo/__init__.py
>  create mode 100644 lib/portage/binrepo/config.py
> 

I had intended to refer to bug 668302 instead of bug 661332:

https://bugs.gentoo.org/668302
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH 1/2] Add binrepos.conf to replace PORTAGE_BINHOST (bug 668334)

2020-09-07 Thread Zac Medico
Support /etc/portage/binrepos.conf as a replacement for the
PORTAGE_BINHOST variable. Behavior is similar to repos.conf,
initially supporting just the sync-uri attribute. Both binrepos.conf
and PORTAGE_BINHOST can be used simultaneously, in the same way that
repos.conf and PORTDIR_OVERLAY can be used simultaneously.

The emerge --info output for binrepos.conf looks like this:

Binary Repositories:

example-binhost
sync-uri: https://example.com/packages

Bug: https://bugs.gentoo.org/668334
Signed-off-by: Zac Medico 
---
 lib/_emerge/actions.py  |  13 ++-
 lib/portage/binrepo/__init__.py |   0
 lib/portage/binrepo/config.py   | 131 
 lib/portage/const.py|   1 +
 lib/portage/dbapi/bintree.py|  14 ++-
 lib/portage/tests/emerge/test_simple.py |  14 ++-
 man/make.conf.5 |   3 +-
 man/portage.5   |  38 +++
 8 files changed, 206 insertions(+), 8 deletions(-)
 create mode 100644 lib/portage/binrepo/__init__.py
 create mode 100644 lib/portage/binrepo/config.py

diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index a4ecfe43d..34344f046 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -32,7 +32,8 @@ portage.proxy.lazyimport.lazyimport(globals(),
 from portage import os
 from portage import shutil
 from portage import _encodings, _unicode_decode
-from portage.const import _DEPCLEAN_LIB_CHECK_DEFAULT
+from portage.binrepo.config import BinRepoConfigLoader
+from portage.const import BINREPOS_CONF_FILE, _DEPCLEAN_LIB_CHECK_DEFAULT
 from portage.dbapi.dep_expand import dep_expand
 from portage.dbapi._expand_new_virt import expand_new_virt
 from portage.dbapi.IndexedPortdb import IndexedPortdb
@@ -1836,6 +1837,16 @@ def action_info(settings, trees, myopts, myfiles):
for repo in repos:
append(repo.info_string())
 
+   binrepos_conf_path = os.path.join(settings['PORTAGE_CONFIGROOT'], 
BINREPOS_CONF_FILE)
+   binrepos_conf = BinRepoConfigLoader((binrepos_conf_path,), settings)
+   if binrepos_conf and any(repo.name for repo in binrepos_conf.values()):
+   append("Binary Repositories:\n")
+   for repo in reversed(list(binrepos_conf.values())):
+   # Omit repos from the PORTAGE_BINHOST variable, since 
they
+   # do not have a name to label them with.
+   if repo.name:
+   append(repo.info_string())
+
installed_sets = sorted(s for s in
root_config.sets['selected'].getNonAtoms() if 
s.startswith(SETPREFIX))
if installed_sets:
diff --git a/lib/portage/binrepo/__init__.py b/lib/portage/binrepo/__init__.py
new file mode 100644
index 0..e69de29bb
diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py
new file mode 100644
index 0..aa3ff7a77
--- /dev/null
+++ b/lib/portage/binrepo/config.py
@@ -0,0 +1,131 @@
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from collections import OrderedDict
+from collections.abc import Mapping
+from hashlib import md5
+
+from portage.localization import _
+from portage.util import _recursive_file_list, writemsg
+from portage.util.configparser import (SafeConfigParser, ConfigParserError,
+   read_configs)
+
+
+class BinRepoConfig:
+   __slots__ = (
+   'name',
+   'name_fallback',
+   'priority',
+   'sync_uri',
+   )
+   def __init__(self, opts):
+   """
+   Create a BinRepoConfig with options in opts.
+   """
+   for k in self.__slots__:
+   setattr(self, k, opts.get(k.replace('_', '-')))
+
+   def info_string(self):
+   """
+   Returns a formatted string containing informations about the 
repository.
+   Used by emerge --info.
+   """
+   indent = " " * 4
+   repo_msg = []
+   repo_msg.append(self.name or self.name_fallback)
+   if self.priority is not None:
+   repo_msg.append(indent + "priority: " + 
str(self.priority))
+   repo_msg.append(indent + "sync-uri: " + self.sync_uri)
+   repo_msg.append("")
+   return "\n".join(repo_msg)
+
+
+class BinRepoConfigLoader(Mapping):
+   def __init__(self, paths, settings):
+   """Load config from files in paths"""
+
+   # Defaults for value interpolation.
+   parser_defaults = {
+   "EPREFIX" : settings["EPREFIX"],
+   "EROOT" : settings["EROOT"],
+ 

[gentoo-portage-dev] [PATCH 2/2] binrepos.conf: support fetchcommand customization (bug 661332)

2020-09-07 Thread Zac Medico
Support customization of fetchcommand and resumecommand in
binrepos.conf, allowing customized authentication mechanisms for
each repository.

Bug: https://bugs.gentoo.org/661332
Signed-off-by: Zac Medico 
---
 lib/_emerge/BinpkgFetcher.py  | 29 +++--
 lib/portage/binrepo/config.py |  2 ++
 lib/portage/dbapi/bintree.py  | 34 +-
 man/portage.5 | 14 ++
 4 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 218d4d2ab..9a96bde28 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -96,14 +96,17 @@ class _BinpkgFetcherProcess(SpawnProcess):
 
# urljoin doesn't work correctly with
# unrecognized protocols like sftp
+   fetchcommand = None
+   resumecommand = None
if bintree._remote_has_index:
-   instance_key = bintree.dbapi._instance_key(pkg.cpv)
-   rel_uri = bintree._remotepkgs[instance_key].get("PATH")
+   remote_metadata = 
bintree._remotepkgs[bintree.dbapi._instance_key(pkg.cpv)]
+   rel_uri = remote_metadata.get("PATH")
if not rel_uri:
rel_uri = pkg.cpv + ".tbz2"
-   remote_base_uri = bintree._remotepkgs[
-   instance_key]["BASE_URI"]
+   remote_base_uri = remote_metadata["BASE_URI"]
uri = remote_base_uri.rstrip("/") + "/" + 
rel_uri.lstrip("/")
+   fetchcommand = remote_metadata.get('FETCHCOMMAND')
+   resumecommand = remote_metadata.get('RESUMECOMMAND')
else:
uri = settings["PORTAGE_BINHOST"].rstrip("/") + \
"/" + pkg.pf + ".tbz2"
@@ -114,13 +117,19 @@ class _BinpkgFetcherProcess(SpawnProcess):
self._async_wait()
return
 
-   protocol = urllib_parse_urlparse(uri)[0]
-   fcmd_prefix = "FETCHCOMMAND"
+   fcmd = None
if resume:
-   fcmd_prefix = "RESUMECOMMAND"
-   fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
-   if not fcmd:
-   fcmd = settings.get(fcmd_prefix)
+   fcmd = resumecommand
+   else:
+   fcmd = fetchcommand
+   if fcmd is None:
+   protocol = urllib_parse_urlparse(uri)[0]
+   fcmd_prefix = "FETCHCOMMAND"
+   if resume:
+   fcmd_prefix = "RESUMECOMMAND"
+   fcmd = settings.get(fcmd_prefix + "_" + 
protocol.upper())
+   if not fcmd:
+   fcmd = settings.get(fcmd_prefix)
 
fcmd_vars = {
"DISTDIR" : os.path.dirname(pkg_path),
diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py
index aa3ff7a77..0c01f6fae 100644
--- a/lib/portage/binrepo/config.py
+++ b/lib/portage/binrepo/config.py
@@ -15,7 +15,9 @@ class BinRepoConfig:
__slots__ = (
'name',
'name_fallback',
+   'fetchcommand',
'priority',
+   'resumecommand',
'sync_uri',
)
def __init__(self, opts):
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index a96183561..ebc6765b6 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -382,10 +382,10 @@ class binarytree:
self._pkgindex_keys.update(["CPV", "SIZE"])
self._pkgindex_aux_keys = \
["BASE_URI", "BDEPEND", "BUILD_ID", 
"BUILD_TIME", "CHOST",
-   "DEFINED_PHASES", "DEPEND", "DESCRIPTION", 
"EAPI",
+   "DEFINED_PHASES", "DEPEND", "DESCRIPTION", 
"EAPI", "FETCHCOMMAND",
"IUSE", "KEYWORDS", "LICENSE", "PDEPEND",
"PKGINDEX_URI", "PROPERTIES", "PROVIDES",
-   "RDEPEND", "repository", "REQUIRES", "RESTRICT",
+   "RDEPEND", "repository", "REQUIRES", 
&quo

[gentoo-portage-dev] [PATCH 0/2] Add binrepos.conf to support fetchcommand customization (bug 661332)

2020-09-07 Thread Zac Medico
Support /etc/portage/binrepos.conf as a replacement for the
PORTAGE_BINHOST variable. Behavior is similar to repos.conf,
initially supporting just the sync-uri attribute. Both binrepos.conf
and PORTAGE_BINHOST can be used simultaneously, in the same way that
repos.conf and PORTDIR_OVERLAY can be used simultaneously.

The emerge --info output for binrepos.conf looks like this:

Binary Repositories:

example-binhost
sync-uri: https://example.com/packages

Support customization of fetchcommand and resumecommand in
binrepos.conf, allowing customized authentication mechanisms for
each repository.

Zac Medico (2):
  Add binrepos.conf to replace PORTAGE_BINHOST (bug 668334)
  binrepos.conf: support fetchcommand customization (bug 661332)

 lib/_emerge/BinpkgFetcher.py|  29 --
 lib/_emerge/actions.py  |  13 ++-
 lib/portage/binrepo/__init__.py |   0
 lib/portage/binrepo/config.py   | 131 
 lib/portage/const.py|   1 +
 lib/portage/dbapi/bintree.py|  48 ++---
 lib/portage/tests/emerge/test_simple.py |  14 ++-
 man/make.conf.5 |   3 +-
 man/portage.5   |  52 ++
 9 files changed, 264 insertions(+), 27 deletions(-)
 create mode 100644 lib/portage/binrepo/__init__.py
 create mode 100644 lib/portage/binrepo/config.py

-- 
2.25.3




[gentoo-portage-dev] [PATCH] make.conf: Treat __* variables as local and do not propagate them.

2020-09-05 Thread Zac Medico
From: Arfrever Frehtes Taifersar Arahesis 

Bug: https://bugs.gentoo.org/740588
Signed-off-by: Arfrever Frehtes Taifersar Arahesis 
Signed-off-by: Zac Medico 
---
 lib/portage/package/ebuild/config.py | 6 ++
 man/make.conf.5  | 4 +++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/portage/package/ebuild/config.py 
b/lib/portage/package/ebuild/config.py
index b62ad3069..a09fdbced 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -370,6 +370,9 @@ class config:
_("Found 2 make.conf files, using both 
'%s' and '%s'") %
tuple(make_conf_paths), noiselevel=-1)
 
+   # __* variables set in make.conf are local and are not 
be propagated.
+   make_conf = {k: v for k, v in make_conf.items() if not 
k.startswith("__")}
+
# Allow ROOT setting to come from make.conf if it's not 
overridden
# by the constructor argument (from the calling 
environment).

locations_manager.set_root_override(make_conf.get("ROOT"))
@@ -621,6 +624,9 @@ class config:
tolerant=tolerant, allow_sourcing=True,
expand=expand_map, recursive=True) or 
{})
 
+   # __* variables set in make.conf are local and are not 
be propagated.
+   mygcfg = {k: v for k, v in mygcfg.items() if not 
k.startswith("__")}
+
# Don't allow the user to override certain variables in 
make.conf
profile_only_variables = 
self.configdict["defaults"].get(
"PROFILE_ONLY_VARIABLES", "").split()
diff --git a/man/make.conf.5 b/man/make.conf.5
index eb812150f..914121b16 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -1,4 +1,4 @@
-.TH "MAKE.CONF" "5" "Jun 2020" "Portage VERSION" "Portage"
+.TH "MAKE.CONF" "5" "Sep 2020" "Portage VERSION" "Portage"
 .SH "NAME"
 make.conf \- custom settings for Portage
 .SH "SYNOPSIS"
@@ -36,6 +36,8 @@ make.defaults to make.globals to make.conf to the environment
 settings.  Clearing these variables requires a clear\-all as in:
 export USE="\-*"
 .br
+__* variables set in make.conf are local and are not be propagated.
+.br
 In order to create per\-package environment settings, refer to
 \fBpackage.env\fR in \fBportage\fR(5).
 .SH "VARIABLES"
-- 
2.25.3




Re: [gentoo-portage-dev] Re: [PATCH v4] env-update: create systemd env configuration if required

2020-09-04 Thread Zac Medico
On 9/4/20 8:39 AM, Florian Schmaus wrote:
> On 9/3/20 7:57 PM, Florian Schmaus wrote:
>> +systemd_profile_env_path = os.path.join(systemd_environment_dir,
>> + "10-gentoo-profile-env.conf")
>> +with open(systemd_profile_env_path, "w") as systemd_profile_env:
> 
> I just noticed that portage has atomic_ofstream(). Would you prefer this
> change to use atomic_ofstream() instead of using "with open()"?

Yes, please do. Also, we can eliminate the earlier os.path.isdir by
using os.makedirs(systemd_environment_dir, exist_ok=True).
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH v2] emerge --search: auto-detect regular expressions (bug 737480)

2020-09-02 Thread Zac Medico
On 9/1/20 11:08 PM, Michał Górny wrote:
> On Tue, 2020-09-01 at 19:59 -0700, Zac Medico wrote:
>> Automatically detect regular expressions when the search string
>> contains any of these regular expression characters or character
>> sequences:
>>
>>   ^ $ * [ ] { } | ? .+
> 
> Isn't this going to break search for gtk+?

It matches the literal sequence '.+', so a 'gtk+' search string
will not trigger it. Since '.' is not allowed in package names,
it shouldn't be very useful except for regular expressions.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH v2] emerge --search: auto-detect regular expressions (bug 737480)

2020-09-02 Thread Zac Medico
On 9/1/20 11:08 PM, Michał Górny wrote:
> On Tue, 2020-09-01 at 19:59 -0700, Zac Medico wrote:
>> Automatically detect regular expressions when the search string
>> contains any of these regular expression characters or character
>> sequences:
>>
>>   ^ $ * [ ] { } | ? .+
> 
> Isn't this going to break search for gtk+?

It matches the literal sequence '.+', so a 'gtk+' search string
will not trigger it. Since '.' is not allowed in package names,
it shouldn't be very useful except for regular expressions.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH v2] emerge --search: auto-detect regular expressions (bug 737480)

2020-09-01 Thread Zac Medico
Automatically detect regular expressions when the search string
contains any of these regular expression characters or character
sequences:

  ^ $ * [ ] { } | ? .+

This simplifies usage, so that users no longer have to remember
to prefix regular expressions with the % character. The new
behavior can be disabled by --regex-search-auto=n, in case the
regular expressions interpretation causes some kind of problem.

Note that fuzzy search and regular expression search are
mutually exclusive, and fuzzy search remains the default for
search strings that do not contain any regular expression
characters.

Bug: https://bugs.gentoo.org/737480
Signed-off-by: Zac Medico 
---
[PATCH v2] Recognize .+ as suggested by Arfrever.

 lib/_emerge/actions.py |  1 +
 lib/_emerge/main.py|  6 ++
 lib/_emerge/search.py  | 12 +++-
 man/emerge.1   | 12 +++-
 4 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index a4ecfe43d..f57269817 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -2036,6 +2036,7 @@ def action_search(root_config, myopts, myfiles, spinner):
search_index=myopts.get("--search-index", "y") != "n",
search_similarity=myopts.get("--search-similarity"),
fuzzy=myopts.get("--fuzzy-search") != "n",
+   regex_auto=myopts.get("--regex-search-auto") != "n",
)
for mysearch in myfiles:
try:
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 975738762..5075f7f57 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -709,6 +709,12 @@ def parse_opts(tmpcmdline, silent=False):
"action" : "store"
},
 
+   "--regex-search-auto": {
+   "help"   : "Enable or disable automatic regular 
expression detection for search actions",
+   "choices": y_or_n,
+   "default": "y",
+   },
+
"--root": {
 "help"   : "specify the target root filesystem for merging 
packages",
 "action" : "store"
diff --git a/lib/_emerge/search.py b/lib/_emerge/search.py
index a59191c1a..61eed0827 100644
--- a/lib/_emerge/search.py
+++ b/lib/_emerge/search.py
@@ -28,7 +28,7 @@ class search:
#
def __init__(self, root_config, spinner, searchdesc,
verbose, usepkg, usepkgonly, search_index=True,
-   search_similarity=None, fuzzy=True):
+   search_similarity=None, fuzzy=True, regex_auto=False):
"""Searches the available and installed packages for the 
supplied search key.
The list of available and installed packages is created at 
object instantiation.
This makes successive searches faster."""
@@ -42,6 +42,7 @@ class search:
self.spinner = None
self.root_config = root_config
self.setconfig = root_config.setconfig
+   self.regex_auto = regex_auto
self.fuzzy = fuzzy
self.search_similarity = (80 if search_similarity is None
else search_similarity)
@@ -259,6 +260,15 @@ class search:
if '/' in self.searchkey:
match_category = 1
fuzzy = False
+
+   if self.regex_auto and not regexsearch and 
re.search(r'[\^\$\*\[\]\{\}\|\?]|\.\+', self.searchkey) is not None:
+   try:
+   re.compile(self.searchkey, re.I)
+   except Exception:
+   pass
+   else:
+   regexsearch = True
+
if regexsearch:
self.searchre=re.compile(self.searchkey,re.I)
else:
diff --git a/man/emerge.1 b/man/emerge.1
index fe7d05a21..c1bcd0220 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -229,7 +229,9 @@ explicitly discarded by running `emaint \-\-fix 
cleanresume` (see
 .BR \-\-search ", " \-s
 Searches for matches of the supplied string in the ebuild repository.
 By default emerge uses a case-insensitive simple search, but you can
-enable a regular expression search by prefixing the search string with %.
+enable a regular expression search by prefixing the search string with %
+(the % prefix can often be omitted if the
+\fB\-\-regex\-search\-auto\fR option is enabled).
 For example, \fBemerge \-\-search "%^kde"\fR searches for any package whose
 name starts with "kde"; \fBemerge \-\-search "%gcc$"\fR searches for

[gentoo-portage-dev] [PATCH] egencache --update-pkg-desc-index: emulate esync --verbose output (bug 737470)

2020-09-01 Thread Zac Medico
When the --verbose flag is given, make --update-pkg-desc-index emulate
esync --verbose output. Example:

 * Searching for changes

 [ N] acct-group/ultimaker (0):  Group for ultimaker
 [ N] acct-user/ultimaker (0):  User for ultimaker
 [ U] www-client/opera (70.0.3728.144):  A fast and secure web browser
 [MU] www-client/opera-developer (72.0.3798.0):  A fast and secure web browser
 [ U] x11-libs/gtksourceview (4.6.1-r1):  A text widget implementing syntax 
highlighting and other features

Bug: https://bugs.gentoo.org/737470
Signed-off-by: Zac Medico 
---
 bin/egencache   | 68 ++---
 man/egencache.1 |  3 +++
 2 files changed, 67 insertions(+), 4 deletions(-)

diff --git a/bin/egencache b/bin/egencache
index 264c600fe..968d5706f 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -35,6 +35,7 @@ else:
 
 signal.signal(debug_signum, debug_signal)
 
+import functools
 import io
 import logging
 import subprocess
@@ -50,8 +51,9 @@ portage._internal_caller = True
 from portage import os, _encodings, _unicode_encode, _unicode_decode
 from _emerge.MetadataRegen import MetadataRegen
 from portage.cache.cache_errors import CacheError, StatCollision
-from portage.cache.index.pkg_desc_index import pkg_desc_index_line_format
+from portage.cache.index.pkg_desc_index import pkg_desc_index_line_format, 
pkg_desc_index_line_read
 from portage.const import TIMESTAMP_FORMAT
+from portage.output import colorize, EOutput
 from portage.package.ebuild._parallel_manifest.ManifestScheduler import 
ManifestScheduler
 from portage.util import cmp_sort_key, writemsg_level
 from portage.util._async.AsyncFunction import AsyncFunction
@@ -131,6 +133,9 @@ def parse_args(args):
common.add_argument("--ignore-default-opts",
action="store_true",
help="do not use the EGENCACHE_DEFAULT_OPTS environment 
variable")
+   common.add_argument("-v", "--verbose",
+   action="count", default=0,
+   help="increase verbosity")
common.add_argument("--write-timestamp",
action="store_true",
help="write metadata/timestamp.chk as required for rsync 
repositories")
@@ -448,13 +453,26 @@ class GenCache:
trg_cache._prune_empty_dirs()
 
 class GenPkgDescIndex:
-   def __init__(self, portdb, output_file):
+   def __init__(self, portdb, output_file, verbose=False):
self.returncode = os.EX_OK
self._portdb = portdb
self._output_file = output_file
+   self._verbose = verbose
 
def run(self):
 
+   display_updates = self._verbose > 0
+   old = {}
+   new = {}
+   if display_updates:
+   try:
+   with open(self._output_file, 'rt', 
encoding=_encodings["repo.content"]) as f:
+   for line in f:
+   pkg_desc = 
pkg_desc_index_line_read(line)
+   old[pkg_desc.cp] = pkg_desc
+   except FileNotFoundError:
+   pass
+
portage.util.ensure_dirs(os.path.dirname(self._output_file))
f = portage.util.atomic_ofstream(self._output_file,
encoding=_encodings["repo.content"])
@@ -466,10 +484,51 @@ class GenPkgDescIndex:
continue
desc, = portdb.aux_get(pkgs[-1], ["DESCRIPTION"])
 
-   f.write(pkg_desc_index_line_format(cp, pkgs, desc))
+   line = pkg_desc_index_line_format(cp, pkgs, desc)
+   f.write(line)
+   if display_updates:
+   new[cp] = pkg_desc_index_line_read(line)
 
f.close()
 
+   if display_updates:
+   out = EOutput()
+   out.einfo("Searching for changes")
+   print("")
+   items = sorted(new.values(), key=lambda pkg_desc: 
pkg_desc.cp)
+   haspkgs = False
+   for pkg_desc in items:
+   masked = False
+   version = 
self._portdb.xmatch("bestmatch-visible", pkg_desc.cp)
+   if not version:
+   version = pkg_desc.cpv_list[-1]
+   masked = True
+   old_versions = old.get(pkg_desc.cp)
+   if old_versions is None or version not in 
old_versions.cpv_list:
+   prefix0 = " "
+ 

[gentoo-portage-dev] [PATCH] emerge --search: auto-detect regular expressions (bug 737480)

2020-08-31 Thread Zac Medico
Automatically detect regular expressions when the search string
contains any of these regular expression characters:

  ^$*[]{}|?

This simplifies usage, so that users no longer have to remember
to prefix regular expressions with the % character. The new
behavior can be disabled by --regex-search-auto=n, in case the
regular expressions interpretation causes some kind of problem.

Note that fuzzy search and regular expression search are
mutually exclusive, and fuzzy search remains the default for
search strings that do not contain any regular expression
characters.

Bug: https://bugs.gentoo.org/737480
Signed-off-by: Zac Medico 
---
 lib/_emerge/actions.py |  1 +
 lib/_emerge/main.py|  6 ++
 lib/_emerge/search.py  | 12 +++-
 man/emerge.1   | 11 ++-
 4 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index a4ecfe43d..f57269817 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -2036,6 +2036,7 @@ def action_search(root_config, myopts, myfiles, spinner):
search_index=myopts.get("--search-index", "y") != "n",
search_similarity=myopts.get("--search-similarity"),
fuzzy=myopts.get("--fuzzy-search") != "n",
+   regex_auto=myopts.get("--regex-search-auto") != "n",
)
for mysearch in myfiles:
try:
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 975738762..5075f7f57 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -709,6 +709,12 @@ def parse_opts(tmpcmdline, silent=False):
"action" : "store"
},
 
+   "--regex-search-auto": {
+   "help"   : "Enable or disable automatic regular 
expression detection for search actions",
+   "choices": y_or_n,
+   "default": "y",
+   },
+
"--root": {
 "help"   : "specify the target root filesystem for merging 
packages",
 "action" : "store"
diff --git a/lib/_emerge/search.py b/lib/_emerge/search.py
index a59191c1a..a23c2db80 100644
--- a/lib/_emerge/search.py
+++ b/lib/_emerge/search.py
@@ -28,7 +28,7 @@ class search:
#
def __init__(self, root_config, spinner, searchdesc,
verbose, usepkg, usepkgonly, search_index=True,
-   search_similarity=None, fuzzy=True):
+   search_similarity=None, fuzzy=True, regex_auto=False):
"""Searches the available and installed packages for the 
supplied search key.
The list of available and installed packages is created at 
object instantiation.
This makes successive searches faster."""
@@ -42,6 +42,7 @@ class search:
self.spinner = None
self.root_config = root_config
self.setconfig = root_config.setconfig
+   self.regex_auto = regex_auto
self.fuzzy = fuzzy
self.search_similarity = (80 if search_similarity is None
else search_similarity)
@@ -259,6 +260,15 @@ class search:
if '/' in self.searchkey:
match_category = 1
fuzzy = False
+
+   if self.regex_auto and not regexsearch and 
re.search(r'[^$*\[\]{}|?]', self.searchkey) is not None:
+   try:
+   re.compile(self.searchkey, re.I)
+   except Exception:
+   pass
+   else:
+   regexsearch = True
+
if regexsearch:
self.searchre=re.compile(self.searchkey,re.I)
else:
diff --git a/man/emerge.1 b/man/emerge.1
index fe7d05a21..f2783e7a1 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -229,7 +229,9 @@ explicitly discarded by running `emaint \-\-fix 
cleanresume` (see
 .BR \-\-search ", " \-s
 Searches for matches of the supplied string in the ebuild repository.
 By default emerge uses a case-insensitive simple search, but you can
-enable a regular expression search by prefixing the search string with %.
+enable a regular expression search by prefixing the search string with %
+(the % prefix can often be omitted if the
+\fB\-\-regex\-search\-auto\fR option is enabled).
 For example, \fBemerge \-\-search "%^kde"\fR searches for any package whose
 name starts with "kde"; \fBemerge \-\-search "%gcc$"\fR searches for any
 package that ends with "gcc"; \fBemerge \-\-search "office"\fR searches 

[gentoo-portage-dev] [PATCH] dephgraph: Allow elimination of highest version after slot conflict (bug 439688)

2020-08-30 Thread Zac Medico
After a slot conflict occurs, allow the highest version to be eliminated
from the graph when appropriate. This is needed for correct behavior in
cases the highest version cannot be installed because an older version
is required by some package. This reverts a change related to bug 531656
from commit a9064d08ef4c92a5d0d1bfb3dc8a01b7850812b0, and that change
no longer appears to be necessary, since the unit tests related to bug
531656 now pass without it.

Due to this change in slot conflict handling, the --changed-slot test
case related to bug 456208 will now fail unless we use an @world update to
trigger rebuilds, therefore fix it to do so.

Bug: https://bugs.gentoo.org/439688
Signed-off-by: Zac Medico 
---
 lib/_emerge/depgraph.py  | 9 -
 .../tests/resolver/test_slot_change_without_revbump.py   | 4 ++--
 lib/portage/tests/resolver/test_slot_conflict_rebuild.py | 3 ---
 3 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index e071be8df..3f864aefc 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1501,15 +1501,6 @@ class depgraph:
 
matched = []
for pkg in conflict:
-   if (pkg is highest_pkg and
-   not highest_pkg.installed and
-   inst_pkg is not None and
-   inst_pkg.sub_slot != 
highest_pkg.sub_slot and
-   not 
self._downgrade_probe(highest_pkg)):
-   # If an upgrade is desired, 
force the highest
-   # version into the graph (bug 
#531656).
-   
non_matching_forced.add(highest_pkg)
-
if atom.match(pkg.with_use(
self._pkg_use_enabled(pkg))) 
and \
not (is_arg_parent and 
pkg.installed):
diff --git a/lib/portage/tests/resolver/test_slot_change_without_revbump.py 
b/lib/portage/tests/resolver/test_slot_change_without_revbump.py
index 5cd8c53d1..e05705671 100644
--- a/lib/portage/tests/resolver/test_slot_change_without_revbump.py
+++ b/lib/portage/tests/resolver/test_slot_change_without_revbump.py
@@ -71,8 +71,8 @@ class SlotChangeWithoutRevBumpTestCase(TestCase):
 
# Test --changed-slot
ResolverPlaygroundTestCase(
-   ["app-arch/libarchive"],
-   options = {"--changed-slot": True, "--usepkg": 
True},
+   ["@world"],
+   options = {"--changed-slot": True, "--usepkg": 
True, "--update": True, "--deep": True},
success = True,
mergelist = ["app-arch/libarchive-3.1.1", 
"kde-base/ark-4.10.0"]),
 
diff --git a/lib/portage/tests/resolver/test_slot_conflict_rebuild.py 
b/lib/portage/tests/resolver/test_slot_conflict_rebuild.py
index b3bcf44d0..d650d42ae 100644
--- a/lib/portage/tests/resolver/test_slot_conflict_rebuild.py
+++ b/lib/portage/tests/resolver/test_slot_conflict_rebuild.py
@@ -454,10 +454,7 @@ class SlotConflictRebuildTestCase(TestCase):
finally:
playground.cleanup()
 
-class SlotConflictRebuildGolangTestCase(TestCase):
-
def testSlotConflictRebuildGolang(self):
-   self.todo = True
 
ebuilds = {
 
-- 
2.25.3




[gentoo-portage-dev] [PATCH] Use cached portage.getpid() function (bug 739540)

2020-08-29 Thread Zac Medico
Use the cached portage.getpid() function to avoid unnecessary syscalls,
and update the cache after each call to os.fork() where the fork may
invoke portage APIs.

Bug: https://bugs.gentoo.org/739540
Signed-off-by: Zac Medico 
---
 bin/quickpkg |  2 +-
 lib/_emerge/EbuildBinpkg.py  |  4 +++-
 lib/_emerge/actions.py   |  2 +-
 lib/portage/_emirrordist/FetchTask.py|  4 ++--
 lib/portage/cache/metadata.py|  4 +++-
 lib/portage/elog/mod_mail_summary.py |  5 ++---
 lib/portage/elog/mod_save_summary.py |  2 +-
 lib/portage/locks.py |  2 +-
 lib/portage/package/ebuild/doebuild.py   |  2 +-
 lib/portage/process.py   | 16 +++-
 lib/portage/tests/locks/test_lock_nonblock.py|  1 +
 .../futures/asyncio/test_wakeup_fd_sigchld.py|  2 +-
 lib/portage/util/__init__.py |  4 ++--
 lib/portage/util/_eventloop/EventLoop.py |  6 +++---
 .../util/_eventloop/asyncio_event_loop.py|  4 ++--
 lib/portage/util/_eventloop/global_event_loop.py |  7 +++
 lib/portage/util/locale.py   |  1 +
 lib/portage/util/movefile.py |  2 +-
 lib/portage/util/socks5.py   |  2 +-
 lib/portage/xpak.py  |  2 +-
 20 files changed, 42 insertions(+), 32 deletions(-)

diff --git a/bin/quickpkg b/bin/quickpkg
index be7d1d7af..a171b3bd5 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -111,7 +111,7 @@ def quickpkg_atom(options, infos, arg, eout):
vardb.aux_update(cpv, update_metadata)
xpdata = xpak.xpak(dblnk.dbdir)
binpkg_tmpfile = os.path.join(bintree.pkgdir,
-   cpv + ".tbz2." + str(os.getpid()))
+   cpv + ".tbz2." + str(portage.getpid()))
ensure_dirs(os.path.dirname(binpkg_tmpfile))
binpkg_compression = settings.get("BINPKG_COMPRESS", 
"bzip2")
try:
diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py
index 6e098eb8a..879b3a9aa 100644
--- a/lib/_emerge/EbuildBinpkg.py
+++ b/lib/_emerge/EbuildBinpkg.py
@@ -3,6 +3,8 @@
 
 from _emerge.CompositeTask import CompositeTask
 from _emerge.EbuildPhase import EbuildPhase
+
+import portage
 from portage import os
 
 class EbuildBinpkg(CompositeTask):
@@ -17,7 +19,7 @@ class EbuildBinpkg(CompositeTask):
root_config = pkg.root_config
bintree = root_config.trees["bintree"]
binpkg_tmpfile = os.path.join(bintree.pkgdir,
-   pkg.cpv + ".tbz2." + str(os.getpid()))
+   pkg.cpv + ".tbz2." + str(portage.getpid()))
bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
 
self._binpkg_tmpfile = binpkg_tmpfile
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 063f5d4a0..a4ecfe43d 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -2623,7 +2623,7 @@ def ionice(settings):
if not ionice_cmd:
return
 
-   variables = {"PID" : str(os.getpid())}
+   variables = {"PID" : str(portage.getpid())}
cmd = [varexpand(x, mydict=variables) for x in ionice_cmd]
 
try:
diff --git a/lib/portage/_emirrordist/FetchTask.py 
b/lib/portage/_emirrordist/FetchTask.py
index 457ca2ac6..41f96b962 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -415,7 +415,7 @@ class FetchTask(CompositeTask):
self._fetch_tmp_dir_info = 'distfiles'
distdir = self.config.options.distfiles
 
-   tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % 
os.getpid()
+   tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % 
portage.getpid()
 
variables = {
"DISTDIR": distdir,
@@ -622,7 +622,7 @@ class FetchTask(CompositeTask):
 
head, tail = os.path.split(dest)
hardlink_tmp = os.path.join(head, 
".%s._mirrordist_hardlink_.%s" % \
-   (tail, os.getpid()))
+   (tail, portage.getpid()))
 
try:
try:
diff --git a/lib/portage/cache/metadata.py b/lib/portage/cache/metadata.py
index db81b8ba1..bd1b70fa0 100644
--- a/lib/portage/cache/metadata.py
+++ b/lib/portage/cache/metadata.py
@@ -6,6 +6,8 @@ import errno
 import re
 import stat
 from operator import attrgetter
+
+import portage
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
@@ -122,7 +124,7 @@ class database(flat_hash.database):
 
s

[gentoo-portage-dev] Re: [PATCH 0/2] emerge: Add short -l option for --load-average (bug 699256)

2020-08-16 Thread Zac Medico
On 8/15/20 12:26 PM, Zac Medico wrote:
> Add a short -l option for --load-average just like make(1) has.
> 
> Bug: https://bugs.gentoo.org/699256
> Signed-off-by: Zac Medico 
> 
> Zac Medico (2):
>   emerge: Remove deprecated --changelog option (bug 699256)
>   emerge: Add short -l option for --load-average (bug 699256)
> 
>  lib/_emerge/actions.py |   5 --
>  lib/_emerge/main.py|   5 +-
>  lib/_emerge/resolver/output.py |  32 +---
>  lib/_emerge/resolver/output_helpers.py | 106 -
>  man/emerge.1   |   8 +-
>  5 files changed, 5 insertions(+), 151 deletions(-)
> 

Merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=8ad29c19c1a3ac8529c5c78534769d2771964bcb
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH] repoman: Add --jobs and --load-average options (bug 448462)

2020-08-16 Thread Zac Medico
Add --jobs and --load-average options which allow dependency checks
for multiple profiles to run in parallel. The increase in performance
is non-linear for the number of jobs, but it can be worthwhile
(I measured a 35% decrease in time when running 'repoman -j8 full'
on sys-apps/portage). For the -j1 case (default), all dependency
checks run in the main process as usual, so there is no significant
performance penalty for the default case.

Bug: https://bugs.gentoo.org/448462
Signed-off-by: Zac Medico 
---
 repoman/lib/repoman/argparser.py  |   9 ++
 .../repoman/modules/scan/depend/profile.py| 117 +++---
 repoman/man/repoman.1 |   9 +-
 3 files changed, 116 insertions(+), 19 deletions(-)

diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py
index 670a0e91d..6d545ccca 100644
--- a/repoman/lib/repoman/argparser.py
+++ b/repoman/lib/repoman/argparser.py
@@ -199,6 +199,15 @@ def parse_args(argv, repoman_default_opts):
'--output-style', dest='output_style', choices=output_keys,
help='select output type', default='default')
 
+   parser.add_argument(
+   '-j', '--jobs', dest='jobs', action='store', type=int, 
default=1,
+   help='Specifies the number of jobs (processes) to run 
simultaneously.')
+
+   parser.add_argument(
+   '-l', '--load-average', dest='load_average', action='store', 
type=float, default=None,
+   help='Specifies that no new jobs (processes) should be started 
if there are others '
+   'jobs running and the load average is at least load (a 
floating-point number).')
+
parser.add_argument(
'--mode', dest='mode', choices=mode_keys,
help='specify which mode repoman will run in (default=full)')
diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py 
b/repoman/lib/repoman/modules/scan/depend/profile.py
index 39d8b550c..1eb69422a 100644
--- a/repoman/lib/repoman/modules/scan/depend/profile.py
+++ b/repoman/lib/repoman/modules/scan/depend/profile.py
@@ -2,7 +2,9 @@
 
 
 import copy
+import functools
 import os
+import types
 from pprint import pformat
 
 from _emerge.Package import Package
@@ -15,6 +17,10 @@ from repoman.modules.scan.depend._gen_arches import 
_gen_arches
 from portage.dep import Atom
 from portage.package.ebuild.profile_iuse import iter_iuse_vars
 from portage.util import getconfig
+from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
+from portage.util.futures.executor.fork import ForkExecutor
+from portage.util.futures.iter_completed import async_iter_completed
 
 
 def sort_key(item):
@@ -58,16 +64,14 @@ class ProfileDependsChecks(ScanBase):
def check(self, **kwargs):
'''Perform profile dependant dependency checks
 
-   @param arches:
@param pkg: Package in which we check (object).
@param ebuild: Ebuild which we check (object).
-   @param baddepsyntax: boolean
-   @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom)
@returns: dictionary
'''
ebuild = kwargs.get('ebuild').get()
pkg = kwargs.get('pkg').get()
-   unknown_pkgs, baddepsyntax = _depend_checks(
+
+   ebuild.unknown_pkgs, ebuild.baddepsyntax = _depend_checks(
ebuild, pkg, self.portdb, self.qatracker, 
self.repo_metadata,
self.repo_settings.qadata)
 
@@ -90,8 +94,64 @@ class ProfileDependsChecks(ScanBase):
relevant_profiles.append((keyword, groups, 
prof))
 
relevant_profiles.sort(key=sort_key)
+   ebuild.relevant_profiles = relevant_profiles
+
+   if self.options.jobs <= 1:
+   for task in self._iter_tasks(None, None, ebuild, pkg):
+   task, results = task
+   for result in results:
+   self._check_result(task, result)
+
+   loop = asyncio._wrap_loop()
+   loop.run_until_complete(self._async_check(loop=loop, **kwargs))
+
+   return False
+
+   @coroutine
+   def _async_check(self, loop=None, **kwargs):
+   '''Perform async profile dependant dependency checks
+
+   @param arches:
+   @param pkg: Package in which we check (object).
+   @param ebuild: Ebuild which we check (object).
+   @param baddepsyntax: boolean
+   @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom)
+   @returns: dictionary
+   '''
+   loop = asyncio._wrap_loop(loop)
+   ebuild = kwargs.get('ebuild').get()
+   pkg = kwargs.get('pkg').

[gentoo-portage-dev] [PATCH 0/2] emerge: Add short -l option for --load-average (bug 699256)

2020-08-15 Thread Zac Medico
Add a short -l option for --load-average just like make(1) has.

Bug: https://bugs.gentoo.org/699256
Signed-off-by: Zac Medico 

Zac Medico (2):
  emerge: Remove deprecated --changelog option (bug 699256)
  emerge: Add short -l option for --load-average (bug 699256)

 lib/_emerge/actions.py |   5 --
 lib/_emerge/main.py|   5 +-
 lib/_emerge/resolver/output.py |  32 +---
 lib/_emerge/resolver/output_helpers.py | 106 -
 man/emerge.1   |   8 +-
 5 files changed, 5 insertions(+), 151 deletions(-)

-- 
2.25.3




[gentoo-portage-dev] [PATCH 2/2] emerge: Add short -l option for --load-average (bug 699256)

2020-08-15 Thread Zac Medico
Add a short -l option for --load-average just like make(1) has.

Bug: https://bugs.gentoo.org/699256
Signed-off-by: Zac Medico 
---
 lib/_emerge/main.py | 2 ++
 man/emerge.1| 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 5aac451fe..975738762 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -171,6 +171,7 @@ def insert_optional_args(args):
short_arg_opts = {
'D' : valid_integers,
'j' : valid_integers,
+   'l' : valid_floats,
}
 
# Don't make things like "-kn" expand to "-k n"
@@ -542,6 +543,7 @@ def parse_opts(tmpcmdline, silent=False):
},
 
"--load-average": {
+   "shortopt" : "-l",
 
"help"   :"Specifies that no new builds should be 
started " + \
"if there are other builds running and the load 
average " + \
diff --git a/man/emerge.1 b/man/emerge.1
index 9f77d1fa2..fe7d05a21 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -688,7 +688,7 @@ dependencies are recalculated for remaining packages and 
any with
 unsatisfied dependencies are automatically dropped. Also see
 the related \fB\-\-skipfirst\fR option.
 .TP
-.BR "\-\-load\-average [LOAD]"
+.BR \-l\ [LOAD] ", "  \-\-load\-average[=LOAD]
 Specifies that no new builds should be started if there are other builds
 running and the load average is at least LOAD (a floating-point number).
 With no argument, removes a previous load limit.
-- 
2.25.3




[gentoo-portage-dev] [PATCH 1/2] emerge: Remove deprecated --changelog option (bug 699256)

2020-08-15 Thread Zac Medico
Bug: https://bugs.gentoo.org/699256
Signed-off-by: Zac Medico 
---
 lib/_emerge/actions.py |   5 --
 lib/_emerge/main.py|   3 +-
 lib/_emerge/resolver/output.py |  32 +---
 lib/_emerge/resolver/output_helpers.py | 106 -
 man/emerge.1   |   6 --
 5 files changed, 2 insertions(+), 150 deletions(-)

diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index d259206ad..063f5d4a0 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -2936,11 +2936,6 @@ def run_action(emerge_config):
 
adjust_configs(emerge_config.opts, emerge_config.trees)
 
-   if "--changelog" in emerge_config.opts:
-   writemsg_level(
-   " %s The emerge --changelog (or -l) option is 
deprecated\n" %
-   warn("*"), level=logging.WARNING, noiselevel=-1)
-
if profile_check(emerge_config.trees, emerge_config.action) != os.EX_OK:
return 1
 
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index f4c7e2715..5aac451fe 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -26,7 +26,7 @@ options=[
 "--ask-enter-invalid",
 "--buildpkgonly",
 "--changed-use",
-"--changelog","--columns",
+"--columns",
 "--debug",
 "--digest",
 "--emptytree",
@@ -58,7 +58,6 @@ shortmapping={
 "e":"--emptytree",
 "f":"--fetchonly", "F":"--fetch-all-uri",
 "h":"--help",
-"l":"--changelog",
 "n":"--noreplace", "N":"--newuse",
 "o":"--onlydeps",  "O":"--nodeps",
 "p":"--pretend",   "P":"--prune",
diff --git a/lib/_emerge/resolver/output.py b/lib/_emerge/resolver/output.py
index b6c77ecad..0c90abefb 100644
--- a/lib/_emerge/resolver/output.py
+++ b/lib/_emerge/resolver/output.py
@@ -26,7 +26,7 @@ from portage.versions import best, cpv_getversion
 from _emerge.Blocker import Blocker
 from _emerge.create_world_atom import create_world_atom
 from _emerge.resolver.output_helpers import ( _DisplayConfig, _tree_display,
-   _PackageCounters, _create_use_string, _calc_changelog, PkgInfo)
+   _PackageCounters, _create_use_string, PkgInfo)
 from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice
 
 class Display:
@@ -39,7 +39,6 @@ class Display:
"""
 
def __init__(self):
-   self.changelogs = []
self.print_msg = []
self.blockers = []
self.counters = _PackageCounters()
@@ -561,14 +560,6 @@ class Display:
noiselevel=-1)
 
 
-   def print_changelog(self):
-   """Prints the changelog text to std_out
-   """
-   for chunk in self.changelogs:
-   writemsg_stdout(chunk,
-   noiselevel=-1)
-
-
def get_display_list(self, mylist):
"""Determines the display list to process
 
@@ -668,23 +659,6 @@ class Display:
return pkg_info
 
 
-   def do_changelog(self, pkg, pkg_info):
-   """Processes and adds the changelog text to the master text for 
output
-
-   @param pkg: _emerge.Package.Package instance
-   @param pkg_info: dictionay
-   Modifies self.changelogs
-   """
-   if pkg_info.previous_pkg is not None:
-   ebuild_path_cl = pkg_info.ebuild_path
-   if ebuild_path_cl is None:
-   # binary package
-   ebuild_path_cl = self.portdb.findname(pkg.cpv, 
myrepo=pkg.repo)
-   if ebuild_path_cl is not None:
-   self.changelogs.extend(_calc_changelog(
-   ebuild_path_cl, pkg_info.previous_pkg, 
pkg.cpv))
-
-
def check_system_world(self, pkg):
"""Checks for any occurances of the package in the system or 
world sets
 
@@ -780,8 +754,6 @@ class Display:
pkg_info.attr_display.new_slot = True
if pkg_info.ordered:
self.counters.newslot += 1
-   if self.conf.changelog:
-   self.do_changelog(pkg, pkg_info)
else:
pkg_info.attr_display.new = True
if pkg_info.ordered:
@@ -909,8 +881,6 @@ class Display:
noiselevel=-1)

spawn_no

Re: [gentoo-portage-dev] [PATCH 1/1] Revert "repoman: deprecate netsurf.eclass."

2020-08-14 Thread Zac Medico
On 8/14/20 2:43 PM, Michael Orlitzky wrote:
> This reverts commit a73024729860f9224b8d1660d24c450080b67d9f. This
> eclass was successfully purged from the tree, so the deprecation is no
> longer needed. And eventually, to address an eblit infestation,
> another eclass with the same name will return. The new one will not be
> deprecated.
> 
> Signed-off-by: Michael Orlitzky 
> ---
>  repoman/lib/repoman/modules/linechecks/deprecated/inherit.py | 1 -
>  1 file changed, 1 deletion(-)
> 
> diff --git a/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py 
> b/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py
> index 60410347b..5848a0c37 100644
> --- a/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py
> +++ b/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py
> @@ -33,7 +33,6 @@ class InheritDeprecated(LineCheck):
>   "gst-plugins10": "gstreamer",
>   "ltprune": False,
>   "mono": "mono-env",
> - "netsurf": False,
>   "python": "python-r1 / python-single-r1 / python-any-r1",
>   "ruby": "ruby-ng",
>   "user": "GLEP 81",
> 

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=336df196217d8907788567a0349270348041657e
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] profile masking

2020-08-14 Thread Zac Medico
On 8/14/20 1:42 PM, Zac Medico wrote:
> On 8/14/20 1:08 PM, Ulrich Mueller wrote:
>>>>>>> On Fri, 14 Aug 2020, Zac Medico wrote:
>>
>>> On 8/14/20 8:42 AM, Joakim Tjernlund wrote:
>>>> Yes, I know I can add that in profile/package.mask but I am looking
>>>> for the bigger picture here. This has to stop somehow, there need to
>>>> be something that limits the mask scope to the repo/overlay it is
>>>> defined.
>>
>>> The scope is already limited, but this overlay inherits the mask because
>>> it has the gentoo repo as its master (either implicitly or via a masters
>>> setting in metadata/layout.conf).
>>
>>> I suppose we could add an option to prevent this inheritance.
>>
>> Like an option in repos.conf or layout.conf?
>>
>> The problem I see with this is that preventing inheritance would disable
>> files like license_groups or thirdpartymirrors. So overlays would have
>> to maintain their own versions.
> 
> I've just tested, and it's possible to do this with a -* at the top of
> the overlay/profiles/package.mask file, if we simply modify the the
> grabfile_package function to allow the -* pass through (it currently
> discards it as an invalid atom).

Opened this bug:

https://bugs.gentoo.org/737148
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] profile masking

2020-08-14 Thread Zac Medico
On 8/14/20 1:08 PM, Ulrich Mueller wrote:
>>>>>> On Fri, 14 Aug 2020, Zac Medico wrote:
> 
>> On 8/14/20 8:42 AM, Joakim Tjernlund wrote:
>>> Yes, I know I can add that in profile/package.mask but I am looking
>>> for the bigger picture here. This has to stop somehow, there need to
>>> be something that limits the mask scope to the repo/overlay it is
>>> defined.
> 
>> The scope is already limited, but this overlay inherits the mask because
>> it has the gentoo repo as its master (either implicitly or via a masters
>> setting in metadata/layout.conf).
> 
>> I suppose we could add an option to prevent this inheritance.
> 
> Like an option in repos.conf or layout.conf?
> 
> The problem I see with this is that preventing inheritance would disable
> files like license_groups or thirdpartymirrors. So overlays would have
> to maintain their own versions.

I've just tested, and it's possible to do this with a -* at the top of
the overlay/profiles/package.mask file, if we simply modify the the
grabfile_package function to allow the -* pass through (it currently
discards it as an invalid atom).

>>> I think a good start would be to consider /etc/portage the top
>>> profile and other subprofiles should be able to use the same features
>>> as /etc/portage.
>>>
>>> Portage could start supporting that now, but there would be a while
>>> until one can use them in Gentoo profile.
> 
>> We've got this bug open for the ::repo atom support:
> 
>> https://bugs.gentoo.org/651208
> 
> I still believe that adding ::gentoo to every line in package.mask would
> be the wrong approach to the problem.

Of course, that would be silly. The per-repo package.mask scoping is
actually implemented internally by adding an implicit ::repo to each
atom in the package.mask file.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] profile masking

2020-08-14 Thread Zac Medico
On 8/14/20 8:42 AM, Joakim Tjernlund wrote:
> On Fri, 2020-08-14 at 17:31 +0200, Ulrich Mueller wrote:
>>> On Fri, 14 Aug 2020, Joakim Tjernlund wrote:
>>
>>> When pkgs are masked in the profile, it affects all variants of that
>>> pkgs, even the ones that are in other overlays.
>>> Example:
>>> !!! The following installed packages are masked:
>>> - sys-auth/sssd-::transmode (masked by: package.mask)
>>> /usr/portage/profiles/package.mask:
>>> # Matt Turner  (2020-08-13)
>>> # Masked for testing
>>
>>> My sssd- is now masked.
>>
>>> Could the profile syntax be extended to include syntax allowed in
>>> /etc/portage ? Then one could use the ::gentoo syntax (or so I hope)
>>
>> The :: syntax is Portage specific and doesn't exist in EAPI 7.
>> So there's no chance to get it into the profile dir anytime soon
>> (because that would imply :: to be added to a future EAPI and the
>> top-level profile dir to be bumped to that EAPI).
> 
> Is profile part of EAPI? masks are not defined/used in ebuilds directly.
> 
>>
>> You could override the mask in your overlay's profile/package.mask
>> instead, using an entry with the "-" operator.
> 
> Yes, I know I can add that in profile/package.mask but I am looking for the 
> bigger
> picture here. This has to stop somehow, there need to be something that limits
> the mask scope to the repo/overlay it is defined.

The scope is already limited, but this overlay inherits the mask because
it has the gentoo repo as its master (either implicitly or via a masters
setting in metadata/layout.conf).

I suppose we could add an option to prevent this inheritance.

> I think a good start would be to consider /etc/portage the top profile and 
> other
> subprofiles should be able to use the same features as /etc/portage.
> 
> Portage could start supporting that now, but there would be a while until
> one can use them in Gentoo profile.

We've got this bug open for the ::repo atom support:

https://bugs.gentoo.org/651208
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH] lib/portage/xml/metadata.py: tolerate xml.parsers.expat import failures (bug 736912)

2020-08-12 Thread Zac Medico
Tolerate broken or missing xml support in python.
This reverts a behavior change from commit
935c47d972d986f1822850618442c19c97e300c3.

Fixes: 935c47d972d9 ("lib/portage/xml/metadata.py: fix ungrouped-imports 
w/refactor")
Bug: https://bugs.gentoo.org/736912
Signed-off-by: Zac Medico 
---
 lib/portage/xml/metadata.py | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/portage/xml/metadata.py b/lib/portage/xml/metadata.py
index 808ba2246..875cc4044 100644
--- a/lib/portage/xml/metadata.py
+++ b/lib/portage/xml/metadata.py
@@ -34,7 +34,10 @@ __all__ = ('MetaDataXML', 'parse_metadata_use')
 import re
 import xml.etree.ElementTree as etree
 
-from xml.parsers.expat import ExpatError
+try:
+   from xml.parsers.expat import ExpatError
+except Exception:
+   ExpatError = SyntaxError
 
 from portage import _encodings, _unicode_encode
 from portage.util import cmp_sort_key, unique_everseen
-- 
2.25.3




Re: [gentoo-portage-dev] RFC: conf-update.d hook dir for dispatch-conf

2020-08-08 Thread Zac Medico
On 8/3/20 1:13 AM, Florian Schmaus wrote:
> Portage's dispatch-conf does historically only support RCS for
> configuration file archival. There are currently two unresolved
> feature requests to extend dispatch-conf support for further
> configuration file management tools:
> 
> - bug #260623 git support for dispatch-conf [1]
> - bug #698316 etckeeper support for dispatch-conf [2]
> 
> Extending dispatch-conf by a hook directory, which can be used by
> configuration file management tools to receive notifications about
> dispatch-conf updating a configuration file, appears to be flexible
> approach to satisfy those feature requests (or at least lay the
> groundwork for them).
> 
> Attached is a patch with a prototypical implementation for
> conf-update.d hooks. I am curious whether or not you deem this useful
> enough to get into dispatch-conf.

Yes, and hooks are a great way to allow for customization.

> If my suggested approach is considered sensible, then I am happy to
continue the work on it.

Yes, it looks good. I wonder if we should support hooks that are only
called once per dispatch-conf session, rather than once per file. I see
that your etckeeper hook does not utilize the file argument, so maybe it
would make more sense to call it at the end of the dispatch-conf session?

If it makes sense to have session hooks, then I suppose we can add a
separate directory for them (kind of like how we have separate
repo.postsync.d and postsync.d hooks).

> For example, I want to write a unit test for this. But I am not sure if
> portage's test framework already provides the necessary functionality
> to create test scenarios for the conf-update.d hooks. Any pointers and
> further feedback would be much appreciated.

In lib/portage/tests/emerge/test_simple.py, we execute emerge,
dispatch-conf, and etc-update inside a mock gentoo prefix environment.
Maybe you can copy that file and modify it to do what you want.

> 
> - Florian
> 
> 1: https://bugs.gentoo.org/260623
> 2: https://bugs.gentoo.org/698316
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH 1/3 v2] Add cached portage.getpid() function

2020-08-08 Thread Zac Medico
Since getpid is a syscall, cache results, and update them
via an after fork hook.

Signed-off-by: Zac Medico 
---
 lib/portage/__init__.py   | 16 +
 .../tests/process/test_AsyncFunction.py   | 24 +++
 2 files changed, 40 insertions(+)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 916c93510..4d4b590a8 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -14,6 +14,7 @@ try:
if not hasattr(errno, 'ESTALE'):
# ESTALE may not be defined on some systems, such as interix.
errno.ESTALE = -1
+   import multiprocessing.util
import re
import types
import platform
@@ -368,6 +369,21 @@ _internal_caller = False
 
 _sync_mode = False
 
+class _ForkWatcher:
+   @staticmethod
+   def hook(_ForkWatcher):
+   _ForkWatcher.current_pid = _os.getpid()
+
+_ForkWatcher.hook(_ForkWatcher)
+
+multiprocessing.util.register_after_fork(_ForkWatcher, _ForkWatcher.hook)
+
+def getpid():
+   """
+   Cached version of os.getpid(). ForkProcess updates the cache.
+   """
+   return _ForkWatcher.current_pid
+
 def _get_stdin():
"""
Buggy code in python's multiprocessing/process.py closes sys.stdin
diff --git a/lib/portage/tests/process/test_AsyncFunction.py 
b/lib/portage/tests/process/test_AsyncFunction.py
index 55857026d..3b360e02f 100644
--- a/lib/portage/tests/process/test_AsyncFunction.py
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -3,6 +3,7 @@
 
 import sys
 
+import portage
 from portage import os
 from portage.tests import TestCase
 from portage.util._async.AsyncFunction import AsyncFunction
@@ -36,3 +37,26 @@ class AsyncFunctionTestCase(TestCase):
def testAsyncFunctionStdin(self):
loop = asyncio._wrap_loop()
loop.run_until_complete(self._testAsyncFunctionStdin(loop))
+
+   def _test_getpid_fork(self):
+   """
+   Verify that portage.getpid() cache is updated in a forked child 
process.
+   """
+   loop = asyncio._wrap_loop()
+   proc = AsyncFunction(scheduler=loop, target=portage.getpid)
+   proc.start()
+   proc.wait()
+   self.assertEqual(proc.pid, proc.result)
+
+   def test_getpid_fork(self):
+   self._test_getpid_fork()
+
+   def test_getpid_double_fork(self):
+   """
+   Verify that portage.getpid() cache is updated correctly after
+   two forks.
+   """
+   loop = asyncio._wrap_loop()
+   proc = AsyncFunction(scheduler=loop, 
target=self._test_getpid_fork)
+   proc.start()
+   self.assertEqual(proc.wait(), 0)
-- 
2.25.3




[gentoo-portage-dev] [PATCH 0/3] sqlite: fork safety (bug 736334)

2020-08-07 Thread Zac Medico
Use a separate connection instance for each pid, since
it is not safe to use a connection created in a parent
process.

See: https://www.sqlite.org/howtocorrupt.html
Bug: https://bugs.gentoo.org/736334

Zac Medico (3):
  Add cached portage.getpid() function
  sqlite: add lazy connection init
  sqlite: fork safety (bug 736334)

 lib/portage/__init__.py   | 14 +
 lib/portage/cache/sqlite.py   | 30 +++
 lib/portage/tests/dbapi/test_auxdb.py | 13 ++--
 .../tests/process/test_AsyncFunction.py   | 24 +++
 4 files changed, 74 insertions(+), 7 deletions(-)

-- 
2.25.3




[gentoo-portage-dev] [PATCH 3/3] sqlite: fork safety (bug 736334)

2020-08-07 Thread Zac Medico
Use a separate connection instance for each pid, since
it is not safe to use a connection created in a parent
process.

See: https://www.sqlite.org/howtocorrupt.html
Bug: https://bugs.gentoo.org/736334
Signed-off-by: Zac Medico 
---
 lib/portage/cache/sqlite.py   |  9 +
 lib/portage/tests/dbapi/test_auxdb.py | 13 +++--
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/lib/portage/cache/sqlite.py b/lib/portage/cache/sqlite.py
index 0395dd516..36a4f049e 100644
--- a/lib/portage/cache/sqlite.py
+++ b/lib/portage/cache/sqlite.py
@@ -4,6 +4,7 @@
 import collections
 import re
 
+import portage
 from portage.cache import fs_template
 from portage.cache import cache_errors
 from portage import os
@@ -25,7 +26,7 @@ class database(fs_template.FsBased):
cache_bytes = 1024 * 1024 * 10
 
_connection_info_entry = 
collections.namedtuple('_connection_info_entry',
-   ('connection', 'cursor'))
+   ('connection', 'cursor', 'pid'))
 
def __init__(self, *args, **config):
super(database, self).__init__(*args, **config)
@@ -71,13 +72,13 @@ class database(fs_template.FsBased):
 
@property
def _db_cursor(self):
-   if self._db_connection_info is None:
+   if self._db_connection_info is None or 
self._db_connection_info.pid != portage.getpid():
self._db_init_connection()
return self._db_connection_info.cursor
 
@property
def _db_connection(self):
-   if self._db_connection_info is None:
+   if self._db_connection_info is None or 
self._db_connection_info.pid != portage.getpid():
self._db_init_connection()
return self._db_connection_info.connection
 
@@ -94,7 +95,7 @@ class database(fs_template.FsBased):
connection = self._db_module.connect(
database=_unicode_decode(self._dbpath), 
**connection_kwargs)
cursor = connection.cursor()
-   self._db_connection_info = 
self._connection_info_entry(connection, cursor)
+   self._db_connection_info = 
self._connection_info_entry(connection, cursor, portage.getpid())
self._db_cursor.execute("PRAGMA encoding = %s" % 
self._db_escape_string("UTF-8"))
if not self.readonly and not 
self._ensure_access(self._dbpath):
raise 
cache_errors.InitializationError(self.__class__, "can't ensure perms on %s" % 
self._dbpath)
diff --git a/lib/portage/tests/dbapi/test_auxdb.py 
b/lib/portage/tests/dbapi/test_auxdb.py
index 5c79357d7..7865c3564 100644
--- a/lib/portage/tests/dbapi/test_auxdb.py
+++ b/lib/portage/tests/dbapi/test_auxdb.py
@@ -4,7 +4,8 @@
 from portage.tests import TestCase
 from portage.tests.resolver.ResolverPlayground import ResolverPlayground
 from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
+from portage.util.futures.executor.fork import ForkExecutor
 
 
 class AuxdbTestCase(TestCase):
@@ -61,8 +62,14 @@ class AuxdbTestCase(TestCase):
 
portdb = playground.trees[playground.eroot]["porttree"].dbapi
 
+   def test_func():
+   return 
asyncio._wrap_loop().run_until_complete(self._test_mod_async(
+   ebuilds, ebuild_inherited, 
eclass_defined_phases, eclass_depend, portdb))
+
+   self.assertTrue(test_func())
+
loop = asyncio._wrap_loop()
-   loop.run_until_complete(self._test_mod_async(ebuilds, 
ebuild_inherited, eclass_defined_phases, eclass_depend, portdb))
+   
self.assertTrue(loop.run_until_complete(loop.run_in_executor(ForkExecutor(), 
test_func)))
 
@coroutine
def _test_mod_async(self, ebuilds, ebuild_inherited, 
eclass_defined_phases, eclass_depend, portdb):
@@ -73,3 +80,5 @@ class AuxdbTestCase(TestCase):
self.assertEqual(depend, eclass_depend)
self.assertEqual(eapi, metadata['EAPI'])
self.assertEqual(frozenset(inherited.split()), 
ebuild_inherited)
+
+   coroutine_return(True)
-- 
2.25.3




[gentoo-portage-dev] [PATCH 1/3] Add cached portage.getpid() function

2020-08-07 Thread Zac Medico
Since getpid is a syscall, cache results, and update them
via an after fork hook.

Signed-off-by: Zac Medico 
---
 lib/portage/__init__.py   | 14 +++
 .../tests/process/test_AsyncFunction.py   | 24 +++
 2 files changed, 38 insertions(+)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 916c93510..52902ba7d 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -14,6 +14,7 @@ try:
if not hasattr(errno, 'ESTALE'):
# ESTALE may not be defined on some systems, such as interix.
errno.ESTALE = -1
+   import multiprocessing.util
import re
import types
import platform
@@ -368,6 +369,19 @@ _internal_caller = False
 
 _sync_mode = False
 
+def _fork_watcher(_fork_watcher):
+   _fork_watcher.current_pid = _os.getpid()
+
+_fork_watcher(_fork_watcher)
+
+multiprocessing.util.register_after_fork(_fork_watcher, _fork_watcher)
+
+def getpid():
+   """
+   Cached version of os.getpid(). ForkProcess updates the cache.
+   """
+   return _fork_watcher.current_pid
+
 def _get_stdin():
"""
Buggy code in python's multiprocessing/process.py closes sys.stdin
diff --git a/lib/portage/tests/process/test_AsyncFunction.py 
b/lib/portage/tests/process/test_AsyncFunction.py
index 55857026d..3b360e02f 100644
--- a/lib/portage/tests/process/test_AsyncFunction.py
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -3,6 +3,7 @@
 
 import sys
 
+import portage
 from portage import os
 from portage.tests import TestCase
 from portage.util._async.AsyncFunction import AsyncFunction
@@ -36,3 +37,26 @@ class AsyncFunctionTestCase(TestCase):
def testAsyncFunctionStdin(self):
loop = asyncio._wrap_loop()
loop.run_until_complete(self._testAsyncFunctionStdin(loop))
+
+   def _test_getpid_fork(self):
+   """
+   Verify that portage.getpid() cache is updated in a forked child 
process.
+   """
+   loop = asyncio._wrap_loop()
+   proc = AsyncFunction(scheduler=loop, target=portage.getpid)
+   proc.start()
+   proc.wait()
+   self.assertEqual(proc.pid, proc.result)
+
+   def test_getpid_fork(self):
+   self._test_getpid_fork()
+
+   def test_getpid_double_fork(self):
+   """
+   Verify that portage.getpid() cache is updated correctly after
+   two forks.
+   """
+   loop = asyncio._wrap_loop()
+   proc = AsyncFunction(scheduler=loop, 
target=self._test_getpid_fork)
+   proc.start()
+   self.assertEqual(proc.wait(), 0)
-- 
2.25.3




[gentoo-portage-dev] [PATCH 2/3] sqlite: add lazy connection init

2020-08-07 Thread Zac Medico
Signed-off-by: Zac Medico 
---
 lib/portage/cache/sqlite.py | 29 -
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/lib/portage/cache/sqlite.py b/lib/portage/cache/sqlite.py
index 55ae8f0e5..0395dd516 100644
--- a/lib/portage/cache/sqlite.py
+++ b/lib/portage/cache/sqlite.py
@@ -1,6 +1,7 @@
 # Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+import collections
 import re
 
 from portage.cache import fs_template
@@ -23,6 +24,9 @@ class database(fs_template.FsBased):
# equation: cache_bytes = page_bytes * page_count
cache_bytes = 1024 * 1024 * 10
 
+   _connection_info_entry = 
collections.namedtuple('_connection_info_entry',
+   ('connection', 'cursor'))
+
def __init__(self, *args, **config):
super(database, self).__init__(*args, **config)
self._import_sqlite()
@@ -44,8 +48,8 @@ class database(fs_template.FsBased):
# Set longer timeout for throwing a "database is locked" 
exception.
# Default timeout in sqlite3 module is 5.0 seconds.
config.setdefault("timeout", 15)
-   self._db_init_connection(config)
-   self._db_init_structures()
+   self._config = config
+   self._db_connection_info = None
 
def _import_sqlite(self):
# sqlite3 is optional with >=python-2.5
@@ -65,7 +69,20 @@ class database(fs_template.FsBased):
s = str(s)
return "'%s'" % s.replace("'", "''")
 
-   def _db_init_connection(self, config):
+   @property
+   def _db_cursor(self):
+   if self._db_connection_info is None:
+   self._db_init_connection()
+   return self._db_connection_info.cursor
+
+   @property
+   def _db_connection(self):
+   if self._db_connection_info is None:
+   self._db_init_connection()
+   return self._db_connection_info.connection
+
+   def _db_init_connection(self):
+   config = self._config
self._dbpath = self.location + ".sqlite"
#if os.path.exists(self._dbpath):
#   os.unlink(self._dbpath)
@@ -74,14 +91,16 @@ class database(fs_template.FsBased):
try:
if not self.readonly:
self._ensure_dirs()
-   self._db_connection = self._db_module.connect(
+   connection = self._db_module.connect(
database=_unicode_decode(self._dbpath), 
**connection_kwargs)
-   self._db_cursor = self._db_connection.cursor()
+   cursor = connection.cursor()
+   self._db_connection_info = 
self._connection_info_entry(connection, cursor)
self._db_cursor.execute("PRAGMA encoding = %s" % 
self._db_escape_string("UTF-8"))
if not self.readonly and not 
self._ensure_access(self._dbpath):
raise 
cache_errors.InitializationError(self.__class__, "can't ensure perms on %s" % 
self._dbpath)
self._db_init_cache_size(config["cache_bytes"])
self._db_init_synchronous(config["synchronous"])
+   self._db_init_structures()
except self._db_error as e:
raise cache_errors.InitializationError(self.__class__, 
e)
 
-- 
2.25.3




Re: [gentoo-portage-dev] [PATCH] lib/*: Fix useless-return

2020-08-07 Thread Zac Medico
On 8/7/20 12:06 PM, Aaron Bauman wrote:
> * Python implies such things. Let's drop 'em and be consistent.
> 
> Signed-off-by: Aaron Bauman 
> ---
>  lib/_emerge/EbuildPhase.py | 6 +++---
>  lib/_emerge/resolver/output.py | 5 +
>  lib/portage/elog/mod_custom.py | 1 +
>  lib/portage/elog/mod_echo.py   | 4 ++--
>  lib/portage/elog/mod_mail.py   | 2 ++
>  lib/portage/glsa.py| 3 +++
>  lib/portage/mail.py| 1 +
>  lib/portage/sync/controller.py | 3 ++-
>  lib/portage/util/whirlpool.py  | 2 ++
>  pylintrc   | 1 -
>  10 files changed, 21 insertions(+), 7 deletions(-)

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=28ed2fe2ba42c7e58cb79e7ce991025e6a96
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] lib/*: exempt two core files from ungrouped-imports check

2020-08-06 Thread Zac Medico
On 8/6/20 5:03 PM, Aaron Bauman wrote:
> * These will take some time to refactor/address if at all possible.
> * Turn on ungrouped-imports check for entire repo
> 
> Signed-off-by: Aaron Bauman 
> ---
>  lib/portage/__init__.py |  1 +
>  lib/portage/checksum.py | 15 ---
>  pylintrc|  1 +
>  3 files changed, 10 insertions(+), 7 deletions(-)

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=0dffa633987a49c1a1d7e95edfe6afeb6269d1e5
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] *: drop all import __future__ statements

2020-08-06 Thread Zac Medico
On 8/6/20 6:12 PM, Aaron Bauman wrote:
> * We only support Py3 now.
> 
> Signed-off-by: Aaron Bauman 
> ---
>  bin/archive-conf  |  4 +--
>  bin/check-implicit-pointer-usage.py   |  2 --
>  bin/clean_locks   |  4 +--
>  bin/dispatch-conf |  4 +--
>  bin/dohtml.py |  4 +--
>  bin/doins.py  |  2 --
>  bin/ebuild|  4 +--
>  bin/egencache |  4 +--
>  bin/emaint|  4 +--
>  bin/emerge|  4 +--
>  bin/env-update|  4 +--
>  bin/fixpackages   |  4 +--
>  bin/glsa-check|  4 +--
>  bin/portageq  |  4 +--
>  bin/quickpkg  |  4 +--
>  bin/regenworld|  4 +--
>  lib/_emerge/Scheduler.py  |  2 --
>  lib/_emerge/UserQuery.py  |  2 --
>  lib/_emerge/actions.py|  2 --
>  lib/_emerge/chk_updated_cfg_files.py  |  2 --
>  lib/_emerge/countdown.py  |  4 +--
>  lib/_emerge/depgraph.py   |  2 --
>  lib/_emerge/help.py   |  4 +--
>  lib/_emerge/main.py   |  2 --
>  lib/_emerge/post_emerge.py|  4 +--
>  lib/_emerge/resolver/circular_dependency.py   |  8 +++---
>  lib/_emerge/resolver/package_tracker.py   |  4 +--
>  lib/_emerge/resolver/slot_collision.py|  3 +--
>  lib/_emerge/unmerge.py|  2 --
>  lib/portage/_emirrordist/FetchTask.py |  2 --
>  lib/portage/_global_updates.py|  4 +--
>  lib/portage/_sets/__init__.py |  2 --
>  lib/portage/_sets/dbapi.py|  2 --
>  lib/portage/_sets/libs.py |  4 +--
>  lib/portage/cache/anydbm.py   |  5 ++--
>  lib/portage/cache/sqlite.py   |  3 +--
>  lib/portage/cvstree.py|  2 --
>  lib/portage/dbapi/vartree.py  |  2 --
>  lib/portage/dispatch_conf.py  |  5 +---
>  lib/portage/elog/mod_echo.py  |  4 +--
>  lib/portage/emaint/main.py|  4 +--
>  lib/portage/glsa.py   | 27 +--
>  lib/portage/localization.py   |  4 +--
>  lib/portage/metadata.py   |  2 --
>  lib/portage/module.py |  5 +---
>  lib/portage/news.py   |  2 --
>  lib/portage/output.py |  2 --
>  lib/portage/package/ebuild/fetch.py   |  2 --
>  lib/portage/sync/controller.py|  3 ---
>  lib/portage/sync/old_tree_timestamp.py|  4 +--
>  lib/portage/tests/__init__.py |  4 +--
>  lib/portage/tests/util/test_xattr.py  |  4 +--
>  .../util/_dyn_libs/display_preserved_libs.py  |  6 ++---
>  lib/portage/util/_eventloop/EventLoop.py  |  5 +---
>  .../util/_eventloop/PollSelectAdapter.py  |  5 ++--
>  lib/portage/util/_xattr.py|  4 +--
>  lib/portage/util/locale.py|  4 +--
>  lib/portage/util/movefile.py  |  2 --
>  repoman/bin/repoman   |  5 ++--
>  repoman/lib/repoman/actions.py|  4 +--
>  repoman/lib/repoman/errors.py |  2 --
>  repoman/lib/repoman/gpg.py|  2 --
>  repoman/lib/repoman/main.py   |  5 ++--
>  repoman/lib/repoman/metadata.py   |  2 --
>  .../lib/repoman/modules/commit/repochecks.py  |  2 --
>  .../lib/repoman/modules/scan/ebuild/ebuild.py |  3 +--
>  repoman/lib/repoman/modules/vcs/settings.py   |  3 +--
>  repoman/lib/repoman/modules/vcs/vcs.py|  3 +--
>  repoman/lib/repoman/profile.py|  2 --
>  repoman/lib/repoman/scanner.py|  6 ++---
>  repoman/lib/repoman/tests/__init__.py |  4 +--
>  repoman/lib/repoman/utilities.py  |  4 +--
>  repoman/runtests  |  4 +--
>  repoman/setup.py  |  4 +--
>  runtests  |  4 +--
>  setup.py  |  2 --
>  76 files changed, 74 insertions(+), 212 deletions(-)

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=d695d866e4e7b2ab79264508f1520f89a5585e04
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] lib/*: fix superfluous-parens and enable check

2020-08-06 Thread Zac Medico
On 8/5/20 9:05 PM, Aaron Bauman wrote:
> Signed-off-by: Aaron Bauman 
> ---
>  lib/_emerge/actions.py   | 14 +++---
>  lib/_emerge/resolver/slot_collision.py   |  4 ++--
>  lib/_emerge/stdout_spinner.py|  2 +-
>  lib/_emerge/unmerge.py   |  4 ++--
>  lib/portage/_sets/dbapi.py   |  6 +++---
>  lib/portage/cache/template.py|  2 +-
>  lib/portage/dep/__init__.py  | 10 +-
>  lib/portage/dep/dep_check.py |  2 +-
>  lib/portage/getbinpkg.py |  6 +++---
>  lib/portage/glsa.py  |  2 +-
>  lib/portage/localization.py  |  2 +-
>  lib/portage/locks.py |  2 +-
>  lib/portage/manifest.py  |  2 +-
>  lib/portage/news.py  |  4 ++--
>  lib/portage/package/ebuild/config.py |  2 +-
>  lib/portage/process.py   |  6 +++---
>  lib/portage/sync/controller.py   |  4 ++--
>  lib/portage/sync/modules/rsync/rsync.py  |  8 
>  lib/portage/tests/resolver/ResolverPlayground.py |  2 +-
>  lib/portage/util/__init__.py |  4 ++--
>  lib/portage/util/env_update.py   |  2 +-
>  lib/portage/xpak.py  | 10 +-
>  pylintrc |  1 +
>  23 files changed, 51 insertions(+), 50 deletions(-)

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=897be5c2874ed3edda7b3af84e6ddb34c3095b71
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH 6/6] pylintrc: enable multiple-imports check

2020-08-06 Thread Zac Medico
On 8/5/20 4:51 PM, Aaron Bauman wrote:
> Signed-off-by: Aaron Bauman 
> ---
>  pylintrc | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/pylintrc b/pylintrc
> index b7091516c..5f4ec3278 100644
> --- a/pylintrc
> +++ b/pylintrc
> @@ -19,6 +19,7 @@ enable=
>  misplaced-future,
>  missing-final-newline,
>  mixed-line-endings,
> +multiple-imports,
>  redefined-builtin,
>  reimported,
>  relative-beyond-top-level,
> 

Thanks, merged the whole series:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=bc0e50294a96623fdfcb51d7cea4c3d2a4c5e261
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] Re: [PATCH] lib/portage/repository/config.py: fix trailing-comma-tuple

2020-08-04 Thread Zac Medico
On 8/4/20 7:35 PM, Aaron Bauman wrote:
> * This fixes the one instance of trailing-comma-tuple in the repo
> * A tuple does need created, but lets do so inside of parentheses as
>   expected
> 
> Suggested-By: Sam James 
> Suggested-By: Zac Medico 
> Signed-off-by: Aaron Bauman 
> ---
>  lib/portage/repository/config.py | 2 +-
>  pylintrc | 3 ++-
>  2 files changed, 3 insertions(+), 2 deletions(-)
> 
> diff --git a/lib/portage/repository/config.py 
> b/lib/portage/repository/config.py
> index 0f3e582f8..f7c956dd8 100644
> --- a/lib/portage/repository/config.py
> +++ b/lib/portage/repository/config.py
> @@ -871,7 +871,7 @@ class RepoConfigLoader:
>   continue
>   if repo.masters is None:
>   if self.mainRepo() and repo_name != 
> self.mainRepo().name:
> - repo.masters = self.mainRepo(),
> + repo.masters = (self.mainRepo(),)
>   else:
>   repo.masters = ()
>   else:
> diff --git a/pylintrc b/pylintrc
> index 337311daa..5e13f714c 100644
> --- a/pylintrc
> +++ b/pylintrc
> @@ -21,7 +21,8 @@ enable=
>  redefined-builtin,
>  reimported,
>  relative-beyond-top-level,
> -trailing-newlines,
> +trailing-comma-tuple,
> + trailing-newlines,
>  trailing-whitespace,
>  unexpected-line-ending-format,
>  unnecessary-semicolon,
> 

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=3e0063094ce656a173f46a9a4837260322769f2e
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] DEVELOPING: Update minimum python version

2020-08-04 Thread Zac Medico
On 8/4/20 6:05 PM, Aaron Bauman wrote:
> * Other style/readability fixes
> 
> Signed-off-by: Aaron Bauman 
> ---
>  DEVELOPING | 18 +++---
>  1 file changed, 11 insertions(+), 7 deletions(-)

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=7b3896ccf0001eedc02735b4379bbd7d78bb2449
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] pylintrc: add more module checks

2020-08-04 Thread Zac Medico
On 8/4/20 7:08 AM, Aaron Bauman wrote:
> * Repo is already clear... so let's turn these on
> 
> Signed-off-by: Aaron Bauman 
> ---
>  pylintrc | 25 -
>  1 file changed, 16 insertions(+), 9 deletions(-)
> 
> diff --git a/pylintrc b/pylintrc
> index ad32f70d7..cd76d38dd 100644
> --- a/pylintrc
> +++ b/pylintrc
> @@ -13,16 +13,23 @@
>  
> #disable=no-absolute-import,bad-continuation,C0103,C0114,C0115,E1101,W0201,no-name-in-module
>  disable=all
>  enable=
> - missing-final-newline,
> - mixed-line-endings,
> - redefined-builtin,
> - reimported,
> +cyclic-import,
> +import-error,
> + import-self,
> +misplaced-future,
> +missing-final-newline,
> +mixed-line-endings,
> +redefined-builtin,
> +reimported,
> +relative-beyond-top-level,
>   trailing-newlines,
> - trailing-whitespace,
> - unexpected-line-ending-format,
> - unnecessary-semicolon,
> - unused-import,
> - useless-object-inheritance
> +trailing-whitespace,
> +unexpected-line-ending-format,
> +unnecessary-semicolon,
> +unused-import,
> +useless-import-alias,
> +useless-object-inheritance,
> +wildcard-import
>  
>  # A comma-separated list of package or module names from where C extensions 
> may
>  # be loaded. Extensions are loading into the active Python interpreter and 
> may
> 

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=8b10dc3f5bd1ed85f0944ca6f2e9bb6fdafa7f40
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH 1/4] pylintrc: turn on checks for reimported modules

2020-08-03 Thread Zac Medico
On 8/3/20 7:44 PM, Aaron Bauman wrote:
> * also, really fix the enable module structure
> 
> Signed-off-by: Aaron Bauman 
> ---
>  pylintrc | 20 +++-
>  1 file changed, 11 insertions(+), 9 deletions(-)
> 
> diff --git a/pylintrc b/pylintrc
> index 690941715..ad32f70d7 100644
> --- a/pylintrc
> +++ b/pylintrc
> @@ -12,15 +12,17 @@
>  # --disable=W".
>  
> #disable=no-absolute-import,bad-continuation,C0103,C0114,C0115,E1101,W0201,no-name-in-module
>  disable=all
> -enable=missing-final-newline,
> -   mixed-line-endings,
> -   redefined-builtin,
> -   trailing-newlines,
> -   trailing-whitespace,
> -   unexpected-line-ending-format,
> -   unnecessary-semicolon,
> -   unused-import,
> -   useless-object-inheritance
> +enable=
> + missing-final-newline,
> + mixed-line-endings,
> + redefined-builtin,
> + reimported,
> + trailing-newlines,
> + trailing-whitespace,
> + unexpected-line-ending-format,
> + unnecessary-semicolon,
> + unused-import,
> + useless-object-inheritance
>  
>  # A comma-separated list of package or module names from where C extensions 
> may
>  # be loaded. Extensions are loading into the active Python interpreter and 
> may
> 

Thanks, merged the whole series:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=1c45b36ee1d2062ec56ad56cba194affcdf2ce0e
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH 4/4] pylintrc: enable more checks

2020-08-03 Thread Zac Medico
On 8/3/20 5:16 PM, Aaron Bauman wrote:
> * These checks are already passing... so let's enable them so we do not
>   have regressions
> * Sort the checks alphabetically and break into more manageable
>   structure
> * Display all warnings/errors from pylint by enabling all messages as
>   all listed checks are now passing. This will allow folks to run
>   'pylint' without having to modify the pylintrc.
> 
> Signed-off-by: Aaron Bauman 
> ---
>  pylintrc | 12 ++--
>  1 file changed, 10 insertions(+), 2 deletions(-)
> 
> diff --git a/pylintrc b/pylintrc
> index 7fd05f322..1f6867d21 100644
> --- a/pylintrc
> +++ b/pylintrc
> @@ -12,7 +12,15 @@
>  # --disable=W".
>  
> #disable=no-absolute-import,bad-continuation,C0103,C0114,C0115,E1101,W0201,no-name-in-module
>  disable=all
> -enable=redefined-builtin,useless-object-inheritance,trailing-newlines,unused-import,trailing-whitespace
> +enable=missing-final-newline,
> + mixed-line-endings,
> + redefined-builtin,
> + trailing-newlines,
> + trailing-whitespace,
> + unexpected-line-ending-format,
> + unnecessary-semicolon,
> + useless-object-inheritance,
> + unused-import
>  
>  # A comma-separated list of package or module names from where C extensions 
> may
>  # be loaded. Extensions are loading into the active Python interpreter and 
> may
> @@ -51,7 +59,7 @@ unsafe-load-any-extension=no
>  
>  # Only show warnings with the listed confidence levels. Leave empty to show
>  # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
> -confidence=HIGH
> +confidence=
>  
>  [REPORTS]
>  
> 

Thanks, merged the whole series:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=aa329cafb8f7354a02c4da745ccf9208e0d4e4f8
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH 68/68] pylintrc: turn on trailing-whitespace

2020-08-03 Thread Zac Medico
On 8/3/20 3:43 PM, Aaron Bauman wrote:
> Signed-off-by: Aaron Bauman 
> ---
>  pylintrc | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/pylintrc b/pylintrc
> index 6f0eef386..7fd05f322 100644
> --- a/pylintrc
> +++ b/pylintrc
> @@ -12,7 +12,7 @@
>  # --disable=W".
>  
> #disable=no-absolute-import,bad-continuation,C0103,C0114,C0115,E1101,W0201,no-name-in-module
>  disable=all
> -enable=redefined-builtin,useless-object-inheritance,trailing-newlines,unused-import
> +enable=redefined-builtin,useless-object-inheritance,trailing-newlines,unused-import,trailing-whitespace
>  
>  # A comma-separated list of package or module names from where C extensions 
> may
>  # be loaded. Extensions are loading into the active Python interpreter and 
> may
> 

Thanks, merged the whole series:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=d4c94aeb39e7b17eeb87257771a90f41640ebb24
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH 18/18] pylintrc: enable unused-imports

2020-08-03 Thread Zac Medico
On 8/3/20 1:20 PM, Aaron Bauman wrote:
> * Repo is now clear of unused-imports... so turn it on by default
> 
> Signed-off-by: Aaron Bauman 
> ---
>  pylintrc | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/pylintrc b/pylintrc
> index 19f59c165..6f0eef386 100644
> --- a/pylintrc
> +++ b/pylintrc
> @@ -12,7 +12,7 @@
>  # --disable=W".
>  
> #disable=no-absolute-import,bad-continuation,C0103,C0114,C0115,E1101,W0201,no-name-in-module
>  disable=all
> -enable=redefined-builtin,useless-object-inheritance,trailing-newlines
> +enable=redefined-builtin,useless-object-inheritance,trailing-newlines,unused-import
>  
>  # A comma-separated list of package or module names from where C extensions 
> may
>  # be loaded. Extensions are loading into the active Python interpreter and 
> may
> 

Thanks, merged the whole series:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=a8eb52efa1364044bb8dd426b1691ef19f08f595
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH 26/26] lib/portage/tests/emerge/test_simple.py: drop unused-import

2020-08-03 Thread Zac Medico
On 8/3/20 12:06 PM, Aaron Bauman wrote:
> Signed-off-by: Aaron Bauman 
> ---
>  lib/portage/tests/emerge/test_simple.py | 3 +--
>  1 file changed, 1 insertion(+), 2 deletions(-)

Thanks, merged the whole series:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=e8944b35c070d550a6c552dcd2ba5300efda01d0
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] lib/_emerge/resolver/slot_collision.py: fix redefined-builtin W0622

2020-08-02 Thread Zac Medico
On 8/2/20 7:53 PM, Aaron Bauman wrote:
> * This fixes the referenced warning by renaming the 'id' variable to
>  'name' and the 'type' variable to 'atype'
> * Additional cosmetic/style changes found along the way.
> 
> Signed-off-by: Aaron Bauman 

Thanks, merged with s/atype/ctype/ and s/name/idx/:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=602d2cd487bb48788e9654a7b7fd5a3be34150d9
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] doc/api/conf.py: silence redefined-builtin for copyright

2020-08-02 Thread Zac Medico
On 8/2/20 7:21 PM, Aaron Bauman wrote:
> * This locally silences the W0622 warning for the override of the
>   copyright which is required.
> 
> Signed-off-by: Aaron Bauman 
> ---
>  doc/api/conf.py | 6 +++---
>  1 file changed, 3 insertions(+), 3 deletions(-)
> 
> diff --git a/doc/api/conf.py b/doc/api/conf.py
> index f318ca25d..9522960c8 100644
> --- a/doc/api/conf.py
> +++ b/doc/api/conf.py
> @@ -24,9 +24,9 @@ import portage
>  
>  # -- Project information 
> -
>  
> -project = 'portage'
> -copyright = '2020, Gentoo Authors'
> -author = 'Gentoo Authors'
> +project = 'portage' # pylint: disable=redefined-builtin
> +copyright = '2020, Gentoo Authors' # pylint: disable=redefined-builtin
> +author = 'Gentoo Authors' # pylint: disable=redefined-builtin
>  
>  # The full version, including alpha/beta/rc tags
>  release = str(portage.VERSION)
> 

Thanks, merged the copyright line as discussed in #gentoo-portage:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=099e575a4cc132e01534023585cf73b7ff806b95
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH] cnf/repo.postsync.d/example: add egencache --update-pkg-desc-index (bug 735626)

2020-08-02 Thread Zac Medico
Add an egencache --update-pkg-desc-index example for users
of app-portage/esearch to migrate to.

Bug: https://bugs.gentoo.org/735626
Signed-off-by: Zac Medico 
---
 cnf/repo.postsync.d/example | 15 ++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/cnf/repo.postsync.d/example b/cnf/repo.postsync.d/example
index 533bf719c..6c349ab3c 100644
--- a/cnf/repo.postsync.d/example
+++ b/cnf/repo.postsync.d/example
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Example /etc/portage/repo.postsync.d script. Make it executable (chmod +x) 
for
 # Portage to process it.
 #
@@ -45,6 +45,19 @@ if [ -n "${repository_name}" ]; then
ret=1
fi
fi
+
+   # Regenerate the metadata/pkg_desc_index file if needed. It's not
+   # needed for https://gitweb.gentoo.org/repo/sync/gentoo.git which
+   # provides a freshly generated copy.
+   if [[ ! -e ${repository_path}/metadata/pkg_desc_index || (
+   -d ${repository_path}/metadata/md5-cache &&
+   -n "$(find "${repository_path}/metadata/md5-cache" -type f 
-newer "${repository_path}/metadata/pkg_desc_index" -print -quit)" ) ]]; then
+   if ! egencache --update-pkg-desc-index 
--repo="${repository_name}"
+   then
+   echo "!!! egencache failed!"
+   ret=1
+   fi
+   fi
 fi
 
 # Return explicit status.
-- 
2.25.3




Re: [gentoo-portage-dev] [PATCH] Fix R0205 across all of repo.

2020-07-31 Thread Zac Medico
On 7/31/20 6:33 PM, Aaron Bauman wrote:
> Do not need to inherit object in py3.
> 
> All tests passed.
> 
> Signed-off-by: Aaron Bauman 
> ---
>  bin/chpathtool.py|  2 +-
>  bin/doins.py | 10 +-
>  bin/ebuild-ipc.py|  2 +-
>  bin/egencache|  8 
>  bin/portageq |  2 +-
>  bin/socks5-server.py |  2 +-
>  runtests |  2 +-
>  7 files changed, 14 insertions(+), 14 deletions(-)

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=ab9ce26be2f95cef697f7de35c6c198ffc6a
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] pylint progress

2020-07-30 Thread Zac Medico
On 7/29/20 3:14 PM, Alec Warner wrote:
> Hi,
> 
> Recently I've begun to run pylint on the portage codebase. You can see
> some recent PRs on this[0][1][2]. Most of the linter errors I've
> fixed are what I consider 'fairly trivial'. In general I'm happy to
> disable errors (or instances of errors) in addition to resolving them.
> You can see some of this
> in https://github.com/gentoo/portage/pull/593/files where we just
> blanket disable a bunch of very numerous messages (either because I
> don't expect to ever fix them, or because they are more work that I
> think we should do at the moment.)
> 
> So I see essentially a few choices:
>  - (a) Do we fix errors in certain classes and run the linter as part of
> CI. This means that once we resolve errors of a certain class, we can
> enable that class of check and prevent new occurrences.

Yes, this is an extremely useful feature to included in our CI.

>  - (b) Do we just avoid running the linter as part of CI (perhaps
> because we are not far enough along on this journey) and focus our
> efforts to get to a point where we can do (a) without spamming CI?

I think it will be best if we arrange it so that we're not spammed with
recurring messages in every CI run. Otherwise, useful messages will be
difficult to recognize since they'll be drowned in noise.

>  - (c) What messages should we bother not fixing at all so we can just
> not work on those classes of error?
> 
> As an example of (c); I believe 'protected-access' is likely intrusive
> to fix and pointless for portage (cat is out of the bag) where as a
> message like W0612(unused-variable) is plausibly something we should fix
> throughout the codebase.

Sounds good.

> Similarly I think "E0602(undefined-variable)"
> is often a bug in how pylint treats our lazy initialized variables, so
> most of these are false positives.

I think it would be nice if we could enable the undefined-variable check.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] portage/README: Update minimum Python version

2020-07-26 Thread Zac Medico
On 7/24/20 11:45 AM, Aaron Bauman wrote:
> * Per commit e59ec1924d6db957a01c828ce294a7675be5b27c Py 2.7 has been
>   dropped and py3.6 is the minimum supported version
> 
> Bug: https://bugs.gentoo.org/731114
> Signed-off-by: Aaron Bauman 
> ---
>  README | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/README b/README
> index cae987242..54b12f8ce 100644
> --- a/README
> +++ b/README
> @@ -10,7 +10,7 @@ other package managers.
>  Dependencies
>  
>  
> -Python and Bash should be the only hard dependencies. Python 2.7 is the
> +Python and Bash should be the only hard dependencies. Python 3.6 is the
>  minimum supported version.
>  
>  Native Extensions
> 

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=a2b7277ea46b8878ce079fc560a54c59006e9d0b
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH] MergeProcess: replace os.fork with multiprocessing.Process (bug 730192)

2020-07-19 Thread Zac Medico
Fix the MergeProcess _spawn method to call the superclass _spawn
method, in order to replace os.fork with multiprocessing.Process,
promoting a healthy state for the forked interpreter.

Bug: https://bugs.gentoo.org/730192
Signed-off-by: Zac Medico 
---
 lib/portage/dbapi/_MergeProcess.py | 102 -
 1 file changed, 28 insertions(+), 74 deletions(-)

diff --git a/lib/portage/dbapi/_MergeProcess.py 
b/lib/portage/dbapi/_MergeProcess.py
index 274ef586f..adf078cb2 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -3,9 +3,6 @@
 
 import io
 import platform
-import signal
-import sys
-import traceback
 
 import fcntl
 import portage
@@ -24,7 +21,7 @@ class MergeProcess(ForkProcess):
'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild',
'mydbapi', 'postinst_failure', 'prev_mtimes', 'unmerge',
'_elog_reader_fd',
-   '_buf', '_elog_keys', '_locked_vdb')
+   '_buf', '_counter', '_dblink', '_elog_keys', '_locked_vdb')
 
def _start(self):
# Portage should always call setcpv prior to this
@@ -132,57 +129,31 @@ class MergeProcess(ForkProcess):
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
-   counter = None
if not self.unmerge:
-   counter = self.vartree.dbapi.counter_tick()
-
-   parent_pid = os.getpid()
-   pid = None
-   try:
-   pid = os.fork()
-
-   if pid != 0:
-   if not isinstance(pid, int):
-   raise AssertionError(
-   "fork returned non-integer: %s" 
% (repr(pid),))
-
-   os.close(elog_writer_fd)
-   self._elog_reader_fd = elog_reader_fd
-   self._buf = ""
-   self._elog_keys = set()
-   # Discard messages which will be collected by 
the subprocess,
-   # in order to avoid duplicates (bug #446136).
-   
portage.elog.messages.collect_messages(key=mylink.mycpv)
-
-   # invalidate relevant vardbapi caches
-   if self.vartree.dbapi._categories is not None:
-   self.vartree.dbapi._categories = None
-   self.vartree.dbapi._pkgs_changed = True
-   self.vartree.dbapi._clear_pkg_cache(mylink)
-
-   return [pid]
-
-   os.close(elog_reader_fd)
-
-   # Use default signal handlers in order to avoid problems
-   # killing subprocesses as reported in bug #353239.
-   signal.signal(signal.SIGINT, signal.SIG_DFL)
-   signal.signal(signal.SIGTERM, signal.SIG_DFL)
-
-   # Unregister SIGCHLD handler and wakeup_fd for the 
parent
-   # process's event loop (bug 655656).
-   signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-   try:
-   wakeup_fd = signal.set_wakeup_fd(-1)
-   if wakeup_fd > 0:
-   os.close(wakeup_fd)
-   except (ValueError, OSError):
-   pass
-
-   portage.locks._close_fds()
-   # We don't exec, so use close_fds=False
-   # (see _setup_pipes docstring).
-   portage.process._setup_pipes(fd_pipes, close_fds=False)
+   self._counter = self.vartree.dbapi.counter_tick()
+
+   self._dblink = mylink
+   self._elog_reader_fd = elog_reader_fd
+   pids = super(MergeProcess, self)._spawn(args, fd_pipes, 
**kwargs)
+   os.close(elog_writer_fd)
+   self._buf = ""
+   self._elog_keys = set()
+   # Discard messages which will be collected by the subprocess,
+   # in order to avoid duplicates (bug #446136).
+   portage.elog.messages.collect_messages(key=mylink.mycpv)
+
+   # invalidate relevant vardbapi caches
+   if self.vartree.dbapi._categories is not None:
+   self.vartree.dbapi._categories = None
+   self.vartree.dbapi._pkgs_changed = True
+   self.vartree.dbapi._clear_pkg_cache(mylink)
+
+   return pids
+
+   def _run(self):
+   os.close(self._elog_reader_fd)
+ 

[gentoo-portage-dev] [PATCH v2] ForkProcess: replace os.fork with multiprocessing.Process (bug 730192)

2020-07-17 Thread Zac Medico
Replace os.fork with multiprocessing.Process, in order to leverage
any pre-fork and post-fork interpreter housekeeping that it provides,
promoting a healthy state for the forked interpreter.

Since multiprocessing.Process closes sys.__stdin__, fix relevant
code to use the portage._get_stdin() compatibility function.
In case there's a legitimate need to inherit stdin for things like
PROPERTIES=interactive support, create a temporary duplicate of
fd_pipes[0] when appropriate, and restore sys.stdin and sys.__stdin__
in the subprocess.

Bug: https://bugs.gentoo.org/730192
Signed-off-by: Zac Medico 
---
[PATCH v2]
* Use sentinel for all python versions
* Add _proc_join coroutine for non-blocking join

 lib/portage/process.py |   4 +-
 lib/portage/sync/controller.py |   4 +-
 lib/portage/util/_async/ForkProcess.py | 146 +++--
 3 files changed, 119 insertions(+), 35 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 6af668db4..b7316c89d 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -1,5 +1,5 @@
 # portage.py -- core Portage functionality
-# Copyright 1998-2019 Gentoo Authors
+# Copyright 1998-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 
@@ -107,7 +107,7 @@ def sanitize_fds():
if _set_inheritable is not None:
 
whitelist = frozenset([
-   sys.__stdin__.fileno(),
+   portage._get_stdin().fileno(),
sys.__stdout__.fileno(),
sys.__stderr__.fileno(),
])
diff --git a/lib/portage/sync/controller.py b/lib/portage/sync/controller.py
index c4c72e73a..43bb5d520 100644
--- a/lib/portage/sync/controller.py
+++ b/lib/portage/sync/controller.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2019 Gentoo Authors
+# Copyright 2014-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import print_function
@@ -231,7 +231,7 @@ class SyncManager(object):
# Redirect command stderr to stdout, in order to prevent
# spurious cron job emails (bug 566132).
spawn_kwargs["fd_pipes"] = {
-   0: sys.__stdin__.fileno(),
+   0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stdout__.fileno()
}
diff --git a/lib/portage/util/_async/ForkProcess.py 
b/lib/portage/util/_async/ForkProcess.py
index d84e93833..eb01a6232 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -1,37 +1,123 @@
-# Copyright 2012-2013 Gentoo Foundation
+# Copyright 2012-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+import fcntl
+import functools
+import multiprocessing
 import signal
 import sys
-import traceback
 
 import portage
 from portage import os
+from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine
 from _emerge.SpawnProcess import SpawnProcess
 
 class ForkProcess(SpawnProcess):
 
-   __slots__ = ()
+   __slots__ = ('_proc', '_proc_join_task')
+
+   # Number of seconds between poll attempts for process exit status
+   # (after the sentinel has become ready).
+   _proc_join_interval = 0.1
 
def _spawn(self, args, fd_pipes=None, **kwargs):
"""
-   Fork a subprocess, apply local settings, and call fetch().
+   Override SpawnProcess._spawn to fork a subprocess that calls
+   self._run(). This uses multiprocessing.Process in order to 
leverage
+   any pre-fork and post-fork interpreter housekeeping that it 
provides,
+   promoting a healthy state for the forked interpreter.
"""
-
-   parent_pid = os.getpid()
-   pid = None
+   # Since multiprocessing.Process closes sys.__stdin__, create a
+   # temporary duplicate of fd_pipes[0] so that sys.__stdin__ can
+   # be restored in the subprocess, in case this is needed for
+   # things like PROPERTIES=interactive support.
+   stdin_dup = None
try:
-   pid = os.fork()
+   stdin_fd = fd_pipes.get(0)
+   if stdin_fd is not None and stdin_fd == 
portage._get_stdin().fileno():
+   stdin_dup = os.dup(stdin_fd)
+   fcntl.fcntl(stdin_dup, fcntl.F_SETFD,
+   fcntl.fcntl(stdin_fd, fcntl.F_GETFD))
+   fd_pipes[0] = stdin_dup
+   self._proc = 
multiprocessing.Process(target=self._bootstrap, args=(fd_pipes,))
+   self._proc.start()
+   final

Re: [gentoo-portage-dev] [PATCH] Remove py<3.4 import compatibility

2020-07-17 Thread Zac Medico
On 7/17/20 6:58 AM, Michał Górny wrote:
> ---
>  bin/binhost-snapshot  |  5 +--
>  lib/_emerge/BinpkgFetcher.py  |  5 +--
>  lib/_emerge/BlockerCache.py   |  6 +---
>  lib/portage/cache/anydbm.py   | 31 +---
>  lib/portage/dbapi/bintree.py  |  5 +--
>  lib/portage/dbapi/porttree.py |  6 +---
>  lib/portage/dbapi/vartree.py  |  6 +---
>  lib/portage/getbinpkg.py  | 36 +--
>  lib/portage/glsa.py   |  5 +--
>  lib/portage/package/ebuild/fetch.py   | 12 ++-
>  .../asyncio/test_policy_wrapper_recursion.py  |  8 +
>  lib/portage/tests/util/futures/test_retry.py  |  5 +--
>  lib/portage/tests/util/test_socks5.py | 11 ++
>  lib/portage/tests/util/test_xattr.py  | 14 +---
>  lib/portage/util/__init__.py  |  5 +--
>  .../util/_dyn_libs/PreservedLibsRegistry.py   |  6 +---
>  lib/portage/util/_eventloop/EventLoop.py  | 10 ++
>  .../util/_eventloop/asyncio_event_loop.py |  9 ++---
>  lib/portage/util/_urlopen.py  | 12 ++-
>  lib/portage/util/futures/_asyncio/__init__.py |  7 ++--
>  lib/portage/util/futures/_asyncio/tasks.py|  7 +---
>  lib/portage/util/futures/events.py| 12 +++
>  lib/portage/util/futures/futures.py   | 36 ---
>  lib/portage/util/futures/transports.py|  5 +--
>  lib/portage/util/futures/unix_events.py   | 21 ---
>  25 files changed, 61 insertions(+), 224 deletions(-)

Looks good. Please merge.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH 3/3] Remove support code for Python < 3.4

2020-07-17 Thread Zac Medico
On 7/16/20 9:48 PM, Michał Górny wrote:
> Signed-off-by: Michał Górny 
> ---
>  lib/_emerge/AsynchronousLock.py   | 12 +---
>  lib/_emerge/EbuildMetadataPhase.py| 12 +---
>  lib/_emerge/FifoIpcDaemon.py  | 30 +--
>  lib/_emerge/PipeReader.py | 12 +---
>  lib/_emerge/SpawnProcess.py   | 16 --
>  lib/portage/dbapi/_MergeProcess.py| 10 ---
>  lib/portage/locks.py  | 11 ---
>  lib/portage/process.py|  2 +-
>  lib/portage/util/_async/PipeLogger.py | 10 ---
>  lib/portage/util/_eventloop/EventLoop.py  | 23 --
>  .../util/_eventloop/global_event_loop.py  | 15 --
>  lib/portage/util/futures/_asyncio/__init__.py | 26 +---
>  lib/portage/util/futures/unix_events.py   |  4 +--
>  13 files changed, 20 insertions(+), 163 deletions(-)

The whole series LGTM. Please merge.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH] ForkProcess: replace os.fork with multiprocessing.Process (bug 730192)

2020-07-16 Thread Zac Medico
Replace os.fork with multiprocessing.Process, in order to leverage
any pre-fork and post-fork interpreter housekeeping that it provides,
promoting a healthy state for the forked interpreter.

Since multiprocessing.Process closes sys.__stdin__, fix relevant
code to use the portage._get_stdin() compatibility function.
In case there's a legitimate need to inherit stdin for things like
PROPERTIES=interactive support, create a temporary duplicate of
fd_pipes[0] when appropriate, and restore sys.stdin and sys.__stdin__
in the subprocess.

Bug: https://bugs.gentoo.org/730192
Signed-off-by: Zac Medico 
---
 lib/portage/process.py |   4 +-
 lib/portage/sync/controller.py |   4 +-
 lib/portage/util/_async/ForkProcess.py | 135 +++--
 3 files changed, 108 insertions(+), 35 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index f550bcb30..2affd4d4b 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -1,5 +1,5 @@
 # portage.py -- core Portage functionality
-# Copyright 1998-2019 Gentoo Authors
+# Copyright 1998-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 
@@ -107,7 +107,7 @@ def sanitize_fds():
if _set_inheritable is not None:
 
whitelist = frozenset([
-   sys.__stdin__.fileno(),
+   portage._get_stdin().fileno(),
sys.__stdout__.fileno(),
sys.__stderr__.fileno(),
])
diff --git a/lib/portage/sync/controller.py b/lib/portage/sync/controller.py
index c4c72e73a..43bb5d520 100644
--- a/lib/portage/sync/controller.py
+++ b/lib/portage/sync/controller.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2019 Gentoo Authors
+# Copyright 2014-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import print_function
@@ -231,7 +231,7 @@ class SyncManager(object):
# Redirect command stderr to stdout, in order to prevent
# spurious cron job emails (bug 566132).
spawn_kwargs["fd_pipes"] = {
-   0: sys.__stdin__.fileno(),
+   0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stdout__.fileno()
}
diff --git a/lib/portage/util/_async/ForkProcess.py 
b/lib/portage/util/_async/ForkProcess.py
index d84e93833..ade5c67ad 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -1,6 +1,8 @@
-# Copyright 2012-2013 Gentoo Foundation
+# Copyright 2012-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+import fcntl
+import multiprocessing
 import signal
 import sys
 import traceback
@@ -11,27 +13,100 @@ from _emerge.SpawnProcess import SpawnProcess
 
 class ForkProcess(SpawnProcess):
 
-   __slots__ = ()
+   __slots__ = ('_proc',)
 
def _spawn(self, args, fd_pipes=None, **kwargs):
"""
-   Fork a subprocess, apply local settings, and call fetch().
+   Override SpawnProcess._spawn to fork a subprocess that calls
+   self._run(). This uses multiprocessing.Process in order to 
leverage
+   any pre-fork and post-fork interpreter housekeeping that it 
provides,
+   promoting a healthy state for the forked interpreter.
"""
-
-   parent_pid = os.getpid()
-   pid = None
+   # Since multiprocessing.Process closes sys.__stdin__, create a
+   # temporary duplicate of fd_pipes[0] so that sys.__stdin__ can
+   # be restored in the subprocess, in case this is needed for
+   # things like PROPERTIES=interactive support.
+   stdin_dup = None
try:
-   pid = os.fork()
+   stdin_fd = fd_pipes.get(0)
+   if stdin_fd is not None and stdin_fd == 
portage._get_stdin().fileno():
+   stdin_dup = os.dup(stdin_fd)
+   fcntl.fcntl(stdin_dup, fcntl.F_SETFD,
+   fcntl.fcntl(stdin_fd, fcntl.F_GETFD))
+   fd_pipes[0] = stdin_dup
+   self._proc = 
multiprocessing.Process(target=self._bootstrap, args=(fd_pipes,))
+   self._proc.start()
+   finally:
+   if stdin_dup is not None:
+   os.close(stdin_dup)
+
+   if self._use_proc_sentinel(self._proc):
+   self.scheduler.add_reader(self._proc.sentinel, 
self._proc_sentinel_exit, self._proc)
+
+   return [self._proc.pid]
+
+   def _use_proc_sentinel(self, proc):
+   """
+   Use

Re: [gentoo-portage-dev] [PATCH] Clean up more py2 conditional code

2020-07-16 Thread Zac Medico
On 7/16/20 12:05 PM, Michał Górny wrote:
> Closes: https://github.com/gentoo/portage/pull/575
> Signed-off-by: Michał Górny 
> ---
>  lib/portage/cache/anydbm.py   |  4 +-
>  lib/portage/cache/mappings.py | 45 +---
>  lib/portage/cache/sql_template.py |  5 +-
>  lib/portage/cache/template.py | 12 ++---
>  lib/portage/elog/messages.py  |  3 +-
>  lib/portage/output.py |  3 +-
>  lib/portage/package/ebuild/config.py  | 11 +---
>  .../futures/asyncio/test_subprocess_exec.py   |  4 --
>  lib/portage/tests/util/futures/test_retry.py  |  2 -
>  lib/portage/tests/util/test_socks5.py | 16 ++
>  lib/portage/util/__init__.py  | 53 +--
>  lib/portage/util/_dyn_libs/NeededEntry.py | 10 
>  lib/portage/util/digraph.py   |  3 --
>  lib/portage/util/listdir.py   |  2 -
>  lib/portage/util/whirlpool.py | 25 -
>  lib/portage/xpak.py   |  2 -
>  16 files changed, 52 insertions(+), 148 deletions(-)

Looks good. Please merge.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] Eliminate the most of explicit py3 conditionals

2020-07-16 Thread Zac Medico
On 7/16/20 5:18 AM, Michał Górny wrote:
> Eliminate the most of py2/py3 conditions in the code.  Leave a few
> where the relevant code is unclear, they will be addressed later.
> 
> Closes: https://github.com/gentoo/portage/pull/574
> Signed-off-by: Michał Górny 
> ---
>  bin/check-implicit-pointer-usage.py   | 25 ++
>  bin/chmod-lite.py | 11 ++---
>  bin/dohtml.py | 11 ++---
>  bin/doins.py  |  7 ++-
>  bin/ebuild| 15 +-
>  bin/filter-bash-environment.py| 11 ++---
>  bin/install.py| 19 ---
>  bin/pid-ns-init   | 10 ++--
>  bin/xattr-helper.py   | 32 +++-
>  lib/_emerge/DependencyArg.py  | 10 
>  lib/_emerge/JobStatusDisplay.py   |  4 +-
>  lib/_emerge/Package.py|  8 ---
>  lib/_emerge/PackageVirtualDbapi.py|  3 --
>  lib/_emerge/SequentialTaskQueue.py|  3 --
>  lib/_emerge/TaskSequence.py   |  3 --
>  lib/_emerge/UseFlagDisplay.py |  8 ---
>  lib/_emerge/UserQuery.py  | 17 +++
>  lib/_emerge/actions.py|  2 -
>  lib/_emerge/resolver/DbapiProvidesIndex.py|  3 --
>  lib/_emerge/resolver/output_helpers.py| 16 --
>  lib/_emerge/resolver/slot_collision.py|  8 ---
>  lib/portage/__init__.py   | 49 +++
>  lib/portage/_emirrordist/Config.py|  4 --
>  lib/portage/_selinux.py   | 12 -
>  lib/portage/_sets/base.py |  3 --
>  lib/portage/dbapi/porttree.py |  7 +--
>  lib/portage/dep/__init__.py   | 18 +--
>  lib/portage/dep/soname/SonameAtom.py  |  8 ---
>  lib/portage/elog/mod_save_summary.py  |  2 -
>  lib/portage/elog/mod_syslog.py|  4 --
>  lib/portage/exception.py  | 45 -
>  lib/portage/mail.py   | 49 +++
>  lib/portage/manifest.py   |  7 ---
>  lib/portage/output.py |  4 +-
>  lib/portage/process.py| 16 +-
>  lib/portage/proxy/objectproxy.py  |  6 ---
>  lib/portage/repository/config.py  |  6 ---
>  .../tests/unicode/test_string_format.py   | 44 -
>  lib/portage/util/_ShelveUnicodeWrapper.py | 45 -
>  39 files changed, 102 insertions(+), 453 deletions(-)
>  delete mode 100644 lib/portage/util/_ShelveUnicodeWrapper.py

Looks good. Please merge.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] Eliminate basestring/long/_unicode py3 compat

2020-07-15 Thread Zac Medico
On 7/14/20 11:07 PM, Michał Górny wrote:
> Replace basestring and _unicode hacks with str, and long with int.
> 
> Signed-off-by: Michał Górny 
> ---
>  bin/egencache |  7 +--
>  lib/_emerge/BinpkgFetcher.py  |  5 +-
>  lib/_emerge/BlockerCache.py   | 19 ++-
>  lib/_emerge/BlockerDB.py  |  4 +-
>  lib/_emerge/FakeVartree.py| 11 +---
>  lib/_emerge/JobStatusDisplay.py   |  7 +--
>  lib/_emerge/Package.py| 27 -
>  lib/_emerge/Scheduler.py  |  5 +-
>  lib/_emerge/actions.py| 21 +++
>  lib/_emerge/create_world_atom.py  |  8 +--
>  lib/_emerge/depgraph.py   | 15 ++---
>  lib/_emerge/main.py   |  4 +-
>  lib/_emerge/resolver/output.py| 26 -
>  lib/_emerge/resolver/output_helpers.py|  6 +-
>  lib/_emerge/resolver/slot_collision.py|  6 +-
>  lib/portage/__init__.py   | 12 ++--
>  lib/portage/_emirrordist/MirrorDistTask.py|  7 +--
>  lib/portage/_emirrordist/main.py  |  9 +--
>  lib/portage/_sets/base.py |  7 +--
>  lib/portage/cache/flat_hash.py|  5 +-
>  lib/portage/cache/fs_template.py  |  7 +--
>  lib/portage/cache/index/pkg_desc_index.py | 10 +---
>  lib/portage/cache/metadata.py |  8 +--
>  lib/portage/cache/sqlite.py   |  7 +--
>  lib/portage/cache/template.py | 19 ++-
>  lib/portage/cvstree.py|  3 -
>  lib/portage/dbapi/bintree.py  | 55 ---
>  lib/portage/dbapi/porttree.py |  6 +-
>  lib/portage/dbapi/vartree.py  | 33 +--
>  lib/portage/dep/__init__.py   | 50 +++--
>  lib/portage/eclass_cache.py   | 10 +---
>  lib/portage/elog/__init__.py  |  5 +-
>  lib/portage/elog/mod_echo.py  |  5 +-
>  lib/portage/elog/mod_syslog.py|  5 +-
>  lib/portage/emaint/modules/binhost/binhost.py |  9 +--
>  lib/portage/emaint/modules/sync/sync.py   |  9 +--
>  lib/portage/exception.py  |  9 +--
>  lib/portage/getbinpkg.py  |  5 +-
>  lib/portage/locks.py  | 17 +++---
>  lib/portage/mail.py   |  7 +--
>  lib/portage/manifest.py   | 12 +---
>  lib/portage/package/ebuild/config.py  | 11 ++--
>  lib/portage/package/ebuild/doebuild.py|  8 +--
>  .../package/ebuild/getmaskingstatus.py|  7 +--
>  lib/portage/process.py|  5 +-
>  lib/portage/proxy/lazyimport.py   |  7 +--
>  lib/portage/repository/config.py  |  5 +-
>  lib/portage/sync/getaddrinfo_validate.py  |  6 +-
>  lib/portage/sync/modules/rsync/rsync.py   |  7 +--
>  lib/portage/tests/dep/test_match_from_list.py |  7 +--
>  .../tests/resolver/ResolverPlayground.py  |  9 +--
>  .../tests/unicode/test_string_format.py   |  7 +--
>  lib/portage/update.py | 16 ++
>  lib/portage/util/__init__.py  |  8 +--
>  lib/portage/util/_dyn_libs/LinkageMapELF.py   |  6 +-
>  .../util/_dyn_libs/PreservedLibsRegistry.py   |  7 +--
>  lib/portage/util/_urlopen.py  |  9 +--
>  lib/portage/util/changelog.py | 10 ++--
>  lib/portage/util/compression_probe.py |  4 +-
>  lib/portage/util/configparser.py  |  9 +--
>  lib/portage/util/env_update.py|  7 +--
>  lib/portage/util/install_mask.py  |  9 +--
>  lib/portage/versions.py   | 17 ++
>  lib/portage/xml/metadata.py   | 12 ++--
>  repoman/lib/repoman/__init__.py   |  7 +--
>  repoman/lib/repoman/main.py   |  2 -
>  repoman/lib/repoman/metadata.py   |  5 --
>  .../modules/scan/metadata/ebuild_metadata.py  |  5 +-
>  repoman/lib/repoman/utilities.py  |  3 -
>  69 files changed, 227 insertions(+), 490 deletions(-)

Looks good. Please merge.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] Remove unnecessary time.monotonic() compat

2020-07-14 Thread Zac Medico
On 7/14/20 11:41 AM, Michał Górny wrote:
> time.monotonic() is available since py3.3, so there's no need for
> the compat anymore.
> 
> Signed-off-by: Michał Górny 
> ---
>  lib/portage/dbapi/vartree.py |  7 ++--
>  lib/portage/tests/util/futures/test_retry.py |  8 ++---
>  lib/portage/util/_eventloop/EventLoop.py |  6 ++--
>  lib/portage/util/monotonic.py| 34 
>  4 files changed, 10 insertions(+), 45 deletions(-)
>  delete mode 100644 lib/portage/util/monotonic.py

Looks good. Please merge.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] Add caching to use_reduce, vercmp, and catpkgsplit

2020-07-13 Thread Zac Medico
On 7/12/20 11:30 PM, Chun-Yu Shei wrote:
> Each of these functions is called repeatedly with the same arguments
> many times. Cache sizes were selected to minimize memory use increase,
> while still providing about the same speedup compared to a cache with
> unbounded size. "emerge -uDvpU --with-bdeps=y @world" runtime decreases
> from 44.32s -> 29.94s -- a 48% speedup, while the maximum value of the
> RES column in htop increases from 280 MB -> 290 MB.
> 
> "emerge -ep @world" time slightly decreases from 18.77s -> 17.93, while
> max observed RES value actually decreases from 228 MB -> 214 MB (similar
> values observed across a few before/after runs).
> ---
>  lib/portage/dep/__init__.py | 107 +---
>  lib/portage/versions.py |   3 +
>  2 files changed, 67 insertions(+), 43 deletions(-)

Merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=d9ee5b09664ab2255b62c1d52d554721ef8b716a
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] Add caching to use_reduce, vercmp, and catpkgsplit

2020-07-12 Thread Zac Medico
On 7/9/20 12:03 AM, Chun-Yu Shei wrote:
> +def use_reduce(depstr, uselist=(), masklist=(), matchall=False, 
> excludeall=(), is_src_uri=False, \
> + eapi=None, opconvert=False, flat=False, is_valid_flag=None, 
> token_class=None, matchnone=False,
> + subset=None):
> + """
> + Takes a dep string and reduces the use? conditionals out, leaving an 
> array
> + with subarrays. All redundant brackets are removed.
> +
> + @param depstr: depstring
> + @type depstr: String
> + @param uselist: Sequence of use enabled flags
> + @type uselist: Sequence
> + @param masklist: Sequence of masked flags (always treated as disabled)
> + @type masklist: Sequence
> + @param matchall: Treat all conditionals as active. Used by repoman.
> + @type matchall: Bool
> + @param excludeall: Sequence of flags for which negated conditionals are 
> always treated as inactive.
> + @type excludeall: Sequence
> + @param is_src_uri: Indicates if depstr represents a SRC_URI
> + @type is_src_uri: Bool
> + @param eapi: Indicates the EAPI the dep string has to comply to
> + @type eapi: String
> + @param opconvert: Put every operator as first element into it's 
> argument list
> + @type opconvert: Bool
> + @param flat: Create a flat list of all tokens
> + @type flat: Bool
> + @param is_valid_flag: Function that decides if a given use flag might 
> be used in use conditionals
> + @type is_valid_flag: Function
> + @param token_class: Convert all non operator tokens into this class
> + @type token_class: Class
> + @param matchnone: Treat all conditionals as inactive. Used by 
> digestgen().
> + @type matchnone: Bool
> + @param subset: Select a subset of dependencies conditional on the given 
> flags
> + @type subset: Sequence
> + @rtype: List
> + @return: The use reduced depend array
> + """
> + if isinstance(depstr, list):
> + if portage._internal_caller:
> + warnings.warn(_("Passing paren_reduced dep arrays to %s 
> is deprecated. " + \
> + "Pass the original dep string instead.") % \
> + ('portage.dep.use_reduce',), 
> DeprecationWarning, stacklevel=2)
> + depstr = paren_enclose(depstr)
> +
> + if uselist is not None:
> + uselist = tuple(uselist)
> + if masklist is not None:
> + masklist = tuple(masklist)
> + if excludeall is not None:
> + excludeall = tuple(excludeall)
> + if subset is not None:
> + subset = tuple(subset)

The patch looks great, but maybe it's better if we use frozenset instead
of tuple for these.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH 1/3] Add caching to catpkgsplit function

2020-07-06 Thread Zac Medico
On 7/6/20 11:03 AM, Zac Medico wrote:
> On 7/6/20 10:30 AM, Chun-Yu Shei wrote:
>> I finally got a chance to try Sid's lru_cache suggestion, and the
>> results were really good.  Simply adding it on catpkgsplit and moving
>> the body of use_reduce into a separate function (that accepts tuples
>> instead of unhashable lists/sets) and decorating it with lru_cache
>> gets a similar 40% overall speedup for the upgrade case I tested.  It
>> seems like even a relatively small cache size (1000 entries) gives
>> quite a speedup, even though in the use_reduce case, the cache size
>> eventually reaches almost 20,000 entries if no limit is set.  With
>> these two changes, adding caching to match_from_list didn't seem to
>> make much/any difference.
> 
> That's great!
> 
>> The catch is that lru_cache is only available in Python 3.2, so would
>> it make sense to add a dummy lru_cache implementation for Python < 3.2
>> that does nothing?  There is also a backports-functools-lru-cache
>> package that's already available in the Portage tree, but that would
>> add an additional external dependency.
>>
>> I agree that refactoring could yield an even bigger gain, but
>> hopefully this can be implemented as an interim solution to speed up
>> the common emerge case of resolving upgrades.  I'm happy to submit new
>> patches for this, if someone can suggest how to best handle the Python
>> < 3.2 case. :)
>>
>> Thanks,
>> Chun-Yu
> 
> We can safely drop support for < Python 3.6 at this point. Alternatively
> we could add a compatibility shim for Python 2.7 that does not perform
> any caching, but I really don't think it's worth the trouble to support
> it any longer.

We've dropped Python 2.7, so now the minimum version is Python 3.6.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] travis.yml: drop python 2.7 (bug 731114)

2020-07-06 Thread Zac Medico
On 7/6/20 12:07 PM, Michał Górny wrote:
> On Mon, 2020-07-06 at 11:42 -0700, Zac Medico wrote:
>> It should be pretty safe to drop support for python2.7 at this point.
>>
> 
> We should probably also change the trove classifier to ... Python :: 3
> :: Only
> 

Updated the classifier, and merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=e59ec1924d6db957a01c828ce294a7675be5b27c
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH] travis.yml: drop python 2.7 (bug 731114)

2020-07-06 Thread Zac Medico
It should be pretty safe to drop support for python2.7 at this point.

Bug: https://bugs.gentoo.org/731114
Signed-off-by: Zac Medico 
---
 .travis.yml | 1 -
 tox.ini | 6 ++
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2132c8c87..d2935fdab 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,6 @@
 dist: bionic
 language: python
 python:
-- 2.7
 - 3.6
 - 3.7
 - 3.8
diff --git a/tox.ini b/tox.ini
index 79b5b45cb..050a2c455 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,14 +1,12 @@
 [tox]
-envlist = py27,py36,py37,py38,py39,pypy3
+envlist = py36,py37,py38,py39,pypy3
 skipsdist = True
 
 [testenv]
 deps =
pygost
pyyaml
-   py27,py36,py37,py38,py39,pypy3: lxml!=4.2.0
-   py27: pyblake2
-   py27: pysha3
+   py36,py37,py38,py39,pypy3: lxml!=4.2.0
 setenv =
PYTHONPATH={toxinidir}/lib
 commands =
-- 
2.25.3




Re: [gentoo-portage-dev] [PATCH 1/3] Add caching to catpkgsplit function

2020-07-06 Thread Zac Medico
On 7/6/20 10:30 AM, Chun-Yu Shei wrote:
> I finally got a chance to try Sid's lru_cache suggestion, and the
> results were really good.  Simply adding it on catpkgsplit and moving
> the body of use_reduce into a separate function (that accepts tuples
> instead of unhashable lists/sets) and decorating it with lru_cache
> gets a similar 40% overall speedup for the upgrade case I tested.  It
> seems like even a relatively small cache size (1000 entries) gives
> quite a speedup, even though in the use_reduce case, the cache size
> eventually reaches almost 20,000 entries if no limit is set.  With
> these two changes, adding caching to match_from_list didn't seem to
> make much/any difference.

That's great!

> The catch is that lru_cache is only available in Python 3.2, so would
> it make sense to add a dummy lru_cache implementation for Python < 3.2
> that does nothing?  There is also a backports-functools-lru-cache
> package that's already available in the Portage tree, but that would
> add an additional external dependency.
> 
> I agree that refactoring could yield an even bigger gain, but
> hopefully this can be implemented as an interim solution to speed up
> the common emerge case of resolving upgrades.  I'm happy to submit new
> patches for this, if someone can suggest how to best handle the Python
> < 3.2 case. :)
> 
> Thanks,
> Chun-Yu

We can safely drop support for < Python 3.6 at this point. Alternatively
we could add a compatibility shim for Python 2.7 that does not perform
any caching, but I really don't think it's worth the trouble to support
it any longer.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH 0/2] Use consistent list of booleans & validate verify-commit-signature

2020-07-02 Thread Zac Medico
On 7/2/20 8:50 AM, Wynn Wolf Arbor wrote:
> Hi,
> 
> Whilst configuring a bunch of overlay repositories via repos.conf, I
> discovered that not all boolean options take the same values. Some only
> take 'true' and 'false', others are documented only as 'yes' or 'no',
> but take 'true', 'false', 'yes', and 'no'. This is inconsistent and can
> lead to very confusing outcomes, so I decided to write a patch.
> 
> I came across https://bugs.gentoo.org/703698 whilst working on this, and
> since my work was tangentially related, decided to fix that as well.
> 
> Wynn Wolf Arbor (2):
>   repos.conf: Use consistent list of values for boolean options
>   git: Verify boolean values passed to sync-git-verify-commit-signature
> 
>  lib/portage/repository/config.py |  4 +--
>  lib/portage/sync/modules/git/__init__.py | 11 
>  lib/portage/sync/modules/git/git.py  |  2 +-
>  lib/portage/sync/modules/rsync/rsync.py  |  2 +-
>  man/portage.5| 45 
> +++-
>  5 files changed, 36 insertions(+), 28 deletions(-)


Thanks! I've merged both patches, with a couple of additional lower()
calls for case insensitivity:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=141ef203661248a2c29945f8c6770ce0c242eaf0
https://gitweb.gentoo.org/proj/portage.git/commit/?id=33b08baff4825bf84f639cf213de92ed36f76771
-- 
Thanks,
Zac





signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] Re: Speeding up Tree Verification

2020-06-30 Thread Zac Medico
On 6/30/20 10:29 AM, Sid Spry wrote:
> On Mon, Jun 29, 2020, at 9:34 PM, Zac Medico wrote:
>> On 6/29/20 7:15 PM, Sid Spry wrote:
>>> On Mon, Jun 29, 2020, at 9:13 PM, Sid Spry wrote:
>>>> Hello,
>>>>
>>>> I have some runnable pseudocode outlining a faster tree verification 
>>>> algorithm.
>>>
>>> Ah, right. It's worth noting that even faster than this algorithm is simply 
>>> verifying
>>> a .tar.xz. Is that totally off the table? I realize it doesn't fit every 
>>> usecase, but it
>>> seems to be faster in both sync and verification time.
>>
>> We've already got support for that with sync-type = webrsync. However, I
>> imagine sync-type = git is even better. All of the types are covered here:
>>
>> https://wiki.gentoo.org/wiki/Portage_Security
> 
> I'm being warned right now that webrsync-gpg is being deprecated; I've been 
> using
> it. It is, amazingly, faster than a typical rsync and may be faster than a 
> git pull though.

Yeah webrsync-gpg is deprecated but the replacement is sync-type =
webrsync and verification is enabled by default for that sync-type.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] Re: Speeding up Tree Verification

2020-06-29 Thread Zac Medico
On 6/29/20 7:15 PM, Sid Spry wrote:
> On Mon, Jun 29, 2020, at 9:13 PM, Sid Spry wrote:
>> Hello,
>>
>> I have some runnable pseudocode outlining a faster tree verification 
>> algorithm.
> 
> Ah, right. It's worth noting that even faster than this algorithm is simply 
> verifying
> a .tar.xz. Is that totally off the table? I realize it doesn't fit every 
> usecase, but it
> seems to be faster in both sync and verification time.

We've already got support for that with sync-type = webrsync. However, I
imagine sync-type = git is even better. All of the types are covered here:

https://wiki.gentoo.org/wiki/Portage_Security
-- 
Thanks,
Zac



[gentoo-portage-dev] Re: [PATCH] ecompress: optimize docompress -x precompressed comparison

2020-06-28 Thread Zac Medico
On 6/28/20 12:54 PM, Zac Medico wrote:
> + LC_COLLATE=C sort -zu "${T}/.ecompress_skip_files" > 
> "${T}/.ecompress_skip_files_sorted"|| die
> + LC_COLLATE=C sort -zu 
> "${T}/.ecompress_had_precompressed" > 
> "${T}/.ecompress_had_precompressed_sorted" || die
> + LC_COLLATE=C comm -z13 
> "${T}/.ecompress_skip_files_sorted" 
> "${T}/.ecompress_had_precompressed_sorted" > 
> "${T}/.ecompress_had_precompressed" || die

I've updated my branch to use \n separators, since posix comm does not
support the -z option.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH] ecompress: optimize docompress -x precompressed comparison

2020-06-28 Thread Zac Medico
Use sort and comm with temporary files in order to compare lists
of docompress -x and precompressed files, since the file lists
can be extremely large. Also strip ${D%/} from paths in order to
reduce length.

Bug: https://bugs.gentoo.org/721516
Suggested-by: Robin H. Johnson 
Signed-off-by: Zac Medico 
---
 bin/ecompress | 29 ++-
 .../tests/resolver/ResolverPlayground.py  |  1 +
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/bin/ecompress b/bin/ecompress
index 60b083834..983a4d1f7 100755
--- a/bin/ecompress
+++ b/bin/ecompress
@@ -19,29 +19,30 @@ while [[ $# -gt 0 ]] ; do
shift
 
skip_dirs=()
-   skip_files=()
+   > "${T}/.ecompress_skip_files" || die
for skip; do
if [[ -d ${ED%/}/${skip#/} ]]; then
skip_dirs+=( "${ED%/}/${skip#/}" )
else
rm -f "${ED%/}/${skip#/}.ecompress" || die
-   skip_files+=("${ED%/}/${skip#/}")
+   printf '%s\0' "${EPREFIX}/${skip#/}" >> 
"${T}/.ecompress_skip_files"
fi
done
 
if [[ ${#skip_dirs[@]} -gt 0 ]]; then
-   while read -r -d ''; do
-   skip_files+=("${REPLY%.ecompress}")
+   while read -r -d '' skip; do
+   skip=${skip%.ecompress}
+   printf '%s\0' "${skip#${D%/}}" >> 
"${T}/.ecompress_skip_files"
done < <(find "${skip_dirs[@]}" -name '*.ecompress' 
-print0 -delete || die)
fi
 
-   if [[ ${#skip_files[@]} -gt 0 && -s 
${T}/.ecompress_had_precompressed ]]; then
-   sed_args=()
-   for f in "${skip_files[@]}"; do
-   sed_args+=("s|^${f}\$||;")
-   done
-   sed_args+=('/^$/d')
-   sed -f - -i "${T}/.ecompress_had_precompressed" <<< 
"${sed_args[@]}" || die
+   if [[ -s ${T}/.ecompress_skip_files && -s 
${T}/.ecompress_had_precompressed ]]; then
+   # Filter skipped files from 
${T}/.ecompress_had_precompressed,
+   # using temporary files since these lists can be 
extremely large.
+   LC_COLLATE=C sort -zu "${T}/.ecompress_skip_files" > 
"${T}/.ecompress_skip_files_sorted"|| die
+   LC_COLLATE=C sort -zu 
"${T}/.ecompress_had_precompressed" > 
"${T}/.ecompress_had_precompressed_sorted" || die
+   LC_COLLATE=C comm -z13 
"${T}/.ecompress_skip_files_sorted" "${T}/.ecompress_had_precompressed_sorted" 
> "${T}/.ecompress_had_precompressed" || die
+   rm -f "${T}/.ecompress_had_precompressed_sorted" 
"${T}/.ecompress_skip_files"{,_sorted}
fi
 
exit 0
@@ -81,7 +82,7 @@ while [[ $# -gt 0 ]] ; do
continue 2
fi
done
-   echo "${path}" >> 
"${T}"/.ecompress_had_precompressed
+   printf '%s\0' "${path#${D%/}}" 
>> "${T}"/.ecompress_had_precompressed || die
;;
esac
 
@@ -195,8 +196,8 @@ if [[ -s ${T}/.ecompress_had_precompressed ]]; then
eqawarn "(manpages, documentation) when automatic compression is used:"
eqawarn
n=0
-   while read -r f; do
-   eqawarn "  ${f#${D%/}}"
+   while read -r -d '' f; do
+   eqawarn "  ${f}"
if [[ $(( n++ )) -eq 10 ]]; then
eqawarn "  ..."
break
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py 
b/lib/portage/tests/resolver/ResolverPlayground.py
index de80a0cc1..ec2e31ae9 100644
--- a/lib/portage/tests/resolver/ResolverPlayground.py
+++ b/lib/portage/tests/resolver/ResolverPlayground.py
@@ -91,6 +91,7 @@ class ResolverPlayground(object):
"chgrp",
"chmod",
"chown",
+   "comm",
"cp",
"egrep",
"env",
-- 
2.25.3




Re: [gentoo-portage-dev] Add caching to a few commonly used functions

2020-06-27 Thread Zac Medico
On 6/27/20 8:12 PM, Michał Górny wrote:
> Dnia June 28, 2020 3:00:00 AM UTC, Zac Medico  napisał(a):
>> On 6/26/20 11:34 PM, Chun-Yu Shei wrote:
>>> Hi,
>>>
>>> I was recently interested in whether portage could be speed up, since
>>> dependency resolution can sometimes take a while on slower machines.
>>> After generating some flame graphs with cProfile and vmprof, I found
>> 3
>>> functions which seem to be called extremely frequently with the same
>>> arguments: catpkgsplit, use_reduce, and match_from_list.  In the
>> first
>>> two cases, it was simple to cache the results in dicts, while
>>> match_from_list was a bit trickier, since it seems to be a
>> requirement
>>> that it return actual entries from the input "candidate_list".  I
>> also
>>> ran into some test failures if I did the caching after the
>>> mydep.unevaluated_atom.use and mydep.repo checks towards the end of
>> the
>>> function, so the caching is only done up to just before that point.
>>>
>>> The catpkgsplit change seems to definitely be safe, and I'm pretty
>> sure
>>> the use_reduce one is too, since anything that could possibly change
>> the
>>> result is hashed.  I'm a bit less certain about the match_from_list
>> one,
>>> although all tests are passing.
>>>
>>> With all 3 patches together, "emerge -uDvpU --with-bdeps=y @world"
>>> speeds up from 43.53 seconds to 30.96 sec -- a 40.6% speedup. 
>> "emerge
>>> -ep @world" is just a tiny bit faster, going from 18.69 to 18.22 sec
>>> (2.5% improvement).  Since the upgrade case is far more common, this
>>> would really help in daily use, and it shaves about 30 seconds off
>>> the time you have to wait to get to the [Yes/No] prompt (from ~90s to
>>> 60s) on my old Sandy Bridge laptop when performing normal upgrades.
>>>
>>> Hopefully, at least some of these patches can be incorporated, and
>> please
>>> let me know if any changes are necessary.
>>>
>>> Thanks,
>>> Chun-Yu
>>
>> Using global variables for caches like these causes a form of memory
>> leak for use cases involving long-running processes that need to work
>> with many different repositories (and perhaps multiple versions of
>> those
>> repositories).
>>
>> There are at least a couple of different strategies that we can use to
>> avoid this form of memory leak:
>>
>> 1) Limit the scope of the caches so that they have some sort of garbage
>> collection life cycle. For example, it would be natural for the
>> depgraph
>> class to have a local cache of use_reduce results, so that the cache
>> can
>> be garbage collected along with the depgraph.
>>
>> 2) Eliminate redundant calls. For example, redundant calls to
>> catpkgslit
>> can be avoided by constructing more _pkg_str instances, since
>> catpkgsplit is able to return early when its argument happens to be a
>> _pkg_str instance.
> 
> I think the weak stuff from the standard library might also be helpful.
> 
> --
> Best regards, 
> Michał Górny
> 

Hmm, maybe weak global caches are an option?
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] Add caching to a few commonly used functions

2020-06-27 Thread Zac Medico
On 6/26/20 11:34 PM, Chun-Yu Shei wrote:
> Hi,
> 
> I was recently interested in whether portage could be speed up, since
> dependency resolution can sometimes take a while on slower machines.
> After generating some flame graphs with cProfile and vmprof, I found 3
> functions which seem to be called extremely frequently with the same
> arguments: catpkgsplit, use_reduce, and match_from_list.  In the first
> two cases, it was simple to cache the results in dicts, while
> match_from_list was a bit trickier, since it seems to be a requirement
> that it return actual entries from the input "candidate_list".  I also
> ran into some test failures if I did the caching after the
> mydep.unevaluated_atom.use and mydep.repo checks towards the end of the
> function, so the caching is only done up to just before that point.
> 
> The catpkgsplit change seems to definitely be safe, and I'm pretty sure
> the use_reduce one is too, since anything that could possibly change the
> result is hashed.  I'm a bit less certain about the match_from_list one,
> although all tests are passing.
> 
> With all 3 patches together, "emerge -uDvpU --with-bdeps=y @world"
> speeds up from 43.53 seconds to 30.96 sec -- a 40.6% speedup.  "emerge
> -ep @world" is just a tiny bit faster, going from 18.69 to 18.22 sec
> (2.5% improvement).  Since the upgrade case is far more common, this
> would really help in daily use, and it shaves about 30 seconds off
> the time you have to wait to get to the [Yes/No] prompt (from ~90s to
> 60s) on my old Sandy Bridge laptop when performing normal upgrades.
> 
> Hopefully, at least some of these patches can be incorporated, and please
> let me know if any changes are necessary.
> 
> Thanks,
> Chun-Yu

Using global variables for caches like these causes a form of memory
leak for use cases involving long-running processes that need to work
with many different repositories (and perhaps multiple versions of those
repositories).

There are at least a couple of different strategies that we can use to
avoid this form of memory leak:

1) Limit the scope of the caches so that they have some sort of garbage
collection life cycle. For example, it would be natural for the depgraph
class to have a local cache of use_reduce results, so that the cache can
be garbage collected along with the depgraph.

2) Eliminate redundant calls. For example, redundant calls to catpkgslit
can be avoided by constructing more _pkg_str instances, since
catpkgsplit is able to return early when its argument happens to be a
_pkg_str instance.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] ecompress: fix "Argument list too long" for sed (bug 727522)

2020-06-24 Thread Zac Medico
On 6/24/20 10:57 AM, Robin H. Johnson wrote:
> On Tue, Jun 23, 2020 at 05:36:14PM -0700, Zac Medico wrote:
>> From: Patrick McLean 
>>
>> Use sed -f to feed commands to sed via stdin, in order to avoid
>> the "Argument list too long" error reported in bug 727522.
> Will this need to move to a tempfile in the near future, for the size of
> sed_args? Maybe do that work now?

Yeah sure. I'm thinking that sort + comm are ideal for this type of
operation in shell code.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH] ecompress: fix "Argument list too long" for sed (bug 727522)

2020-06-23 Thread Zac Medico
From: Patrick McLean 

Use sed -f to feed commands to sed via stdin, in order to avoid
the "Argument list too long" error reported in bug 727522.

Fixes: 5508bf7a6db5 ("ecompress: ignore docompress -x files in precompressed QA 
check (bug 721516)")
Bug: https://bugs.gentoo.org/727522
Signed-off-by: Zac Medico 
---
 bin/ecompress | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/bin/ecompress b/bin/ecompress
index 7aabc8e4c..1ab07ccd7 100755
--- a/bin/ecompress
+++ b/bin/ecompress
@@ -38,9 +38,10 @@ while [[ $# -gt 0 ]] ; do
if [[ ${#skip_files[@]} -gt 0 && -s 
${T}/.ecompress_had_precompressed ]]; then
sed_args=()
for f in "${skip_files[@]}"; do
-   sed_args+=(-e "s|^${f}\$||")
+   sed_args+=("s|^${f}\$||;")
done
-   sed "${sed_args[@]}" -e '/^$/d' -i 
"${T}/.ecompress_had_precompressed" || die
+   sed_args+=('/^$/d')
+   sed -f - -i "${T}/.ecompress_had_precompressed" <<< 
"${sed_args[@]}" || die
fi
 
exit 0
-- 
2.25.3




Re: [gentoo-portage-dev] [PATCH 2/2] Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)

2020-06-22 Thread Zac Medico
On 6/22/20 7:46 AM, Brian Dolbec wrote:
> 
> That's a lot of code...but I couldn't spot anything wrong, so looks good

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=dd69ce742c62b9515cf7ae37e46bcf7f178777db
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH 1/2] PipeLogger: non-blocking write to pipe (bug 709746)

2020-06-19 Thread Zac Medico
Add support to write to a non-blocking pipe instead of a
log file. This is needed for the purposes of bug 709746,
where PipeLogger will write to a pipe that is drained
by anoher PipeLogger instance which is running in the same
process.

Bug: https://bugs.gentoo.org/709746
Signed-off-by: Zac Medico 
---
 lib/portage/tests/process/test_PipeLogger.py | 58 
 lib/portage/util/_async/PipeLogger.py| 73 +++-
 2 files changed, 115 insertions(+), 16 deletions(-)
 create mode 100644 lib/portage/tests/process/test_PipeLogger.py

diff --git a/lib/portage/tests/process/test_PipeLogger.py 
b/lib/portage/tests/process/test_PipeLogger.py
new file mode 100644
index 0..2bd94cf39
--- /dev/null
+++ b/lib/portage/tests/process/test_PipeLogger.py
@@ -0,0 +1,58 @@
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from portage import os
+from portage.tests import TestCase
+from portage.util._async.PipeLogger import PipeLogger
+from portage.util.futures import asyncio
+from portage.util.futures._asyncio.streams import _reader, _writer
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
+from portage.util.futures.unix_events import _set_nonblocking
+
+
+class PipeLoggerTestCase(TestCase):
+
+   @coroutine
+   def _testPipeLoggerToPipe(self, test_string, loop=None):
+   """
+   Test PipeLogger writing to a pipe connected to a PipeReader.
+   This verifies that PipeLogger does not deadlock when writing
+   to a pipe that's drained by a PipeReader running in the same
+   process (requires non-blocking write).
+   """
+
+   input_fd, writer_pipe = os.pipe()
+   _set_nonblocking(writer_pipe)
+   writer_pipe = os.fdopen(writer_pipe, 'wb', 0)
+   writer = asyncio.ensure_future(_writer(writer_pipe, 
test_string.encode('ascii'), loop=loop), loop=loop)
+   writer.add_done_callback(lambda writer: writer_pipe.close())
+
+   pr, pw = os.pipe()
+
+   consumer = PipeLogger(background=True,
+   input_fd=input_fd,
+   log_file_path=os.fdopen(pw, 'wb', 0),
+   scheduler=loop)
+   consumer.start()
+
+   # Before starting the reader, wait here for a moment, in order
+   # to exercise PipeLogger's handling of EAGAIN during write.
+   yield asyncio.wait([writer], timeout=0.01)
+
+   reader = _reader(pr, loop=loop)
+   yield writer
+   content = yield reader
+   yield consumer.async_wait()
+
+   self.assertEqual(consumer.returncode, os.EX_OK)
+
+   coroutine_return(content.decode('ascii', 'replace'))
+
+   def testPipeLogger(self):
+   loop = asyncio._wrap_loop()
+
+   for x in (1, 2, 5, 6, 7, 8, 2**5, 2**10, 2**12, 2**13, 2**14, 
2**17, 2**17 + 1):
+   test_string = x * "a"
+   output = 
loop.run_until_complete(self._testPipeLoggerToPipe(test_string, loop=loop))
+   self.assertEqual(test_string, output,
+   "x = %s, len(output) = %s" % (x, len(output)))
diff --git a/lib/portage/util/_async/PipeLogger.py 
b/lib/portage/util/_async/PipeLogger.py
index a4258f350..ce8afb846 100644
--- a/lib/portage/util/_async/PipeLogger.py
+++ b/lib/portage/util/_async/PipeLogger.py
@@ -8,6 +8,10 @@ import sys
 
 import portage
 from portage import os, _encodings, _unicode_encode
+from portage.util.futures import asyncio
+from portage.util.futures._asyncio.streams import _writer
+from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures.unix_events import _set_nonblocking
 from _emerge.AbstractPollTask import AbstractPollTask
 
 class PipeLogger(AbstractPollTask):
@@ -21,13 +25,16 @@ class PipeLogger(AbstractPollTask):
"""
 
__slots__ = ("input_fd", "log_file_path", "stdout_fd") + \
-   ("_log_file", "_log_file_real")
+   ("_io_loop_task", "_log_file", "_log_file_nb", "_log_file_real")
 
def _start(self):
 
log_file_path = self.log_file_path
-   if log_file_path is not None:
-
+   if hasattr(log_file_path, 'write'):
+   self._log_file_nb = True
+   self._log_file = log_file_path
+   _set_nonblocking(self._log_file.fileno())
+   elif log_file_path is not None:
self._log_file = open(_unicode_encode(log_file_path,
encoding=_encodings['fs'], errors='strict'), 
mode='ab')
if log_f

[gentoo-portage-dev] [PATCH 0/2] Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)

2020-06-19 Thread Zac Medico
This variable specifies a command that filters build log output to a
log file. The plan is to extend this to support a separate filter for
tty output in the future.

Previous versions of these patches were affected by bug 716636 which
was due to unsafe remove_reader and remove_writer calls in finally
clauses of the PipeLogger _io_loop coroutine. The remove_reader and
remove_writer calls are now skipped if the corresponding file object
has already been closed (which is normal if the coroutine has been
cancelled). Since this kind of bug is not easy to reproduce, currently
Rick Farina  is testing the patches in order to
verify that they do not trigger emerge hangs like bug 716636.

Bug: https://bugs.gentoo.org/709746
Bug: https://bugs.gentoo.org/716636

Zac Medico (2):
  PipeLogger: non-blocking write to pipe (bug 709746)
  Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)

 lib/_emerge/AbstractEbuildProcess.py  |   3 +-
 lib/_emerge/BinpkgFetcher.py  |   3 +-
 lib/_emerge/EbuildFetcher.py  |   3 +-
 lib/_emerge/EbuildPhase.py|  47 ++--
 lib/_emerge/SpawnProcess.py   |  58 +++---
 lib/portage/dbapi/_MergeProcess.py|   3 +-
 .../ebuild/_config/special_env_vars.py|   8 +-
 lib/portage/tests/process/test_PipeLogger.py  |  58 ++
 lib/portage/util/_async/BuildLogger.py| 109 ++
 lib/portage/util/_async/PipeLogger.py |  73 +---
 lib/portage/util/_async/SchedulerInterface.py |  32 -
 man/make.conf.5   |   7 +-
 12 files changed, 358 insertions(+), 46 deletions(-)
 create mode 100644 lib/portage/tests/process/test_PipeLogger.py
 create mode 100644 lib/portage/util/_async/BuildLogger.py

-- 
2.25.3




[gentoo-portage-dev] [PATCH 2/2] Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)

2020-06-19 Thread Zac Medico
This variable specifies a command that filters build log output to a
log file. The plan is to extend this to support a separate filter for
tty output in the future.

In order to enable the EbuildPhase class to write elog messages to
the build log with PORTAGE_LOG_FILTER_FILE_CMD support, convert its
_elog method to a coroutine, and add a SchedulerInterface async_output
method for it to use.

Use a new BuildLogger class to manage log output (with or without a
filter command), with compression support provided by PipeLogger.
BuildLogger has a stdin property which provides access to a writable
binary file stream (refers to a pipe) that log content is written to.

Bug: https://bugs.gentoo.org/709746
Signed-off-by: Zac Medico 
---
 lib/_emerge/AbstractEbuildProcess.py  |   3 +-
 lib/_emerge/BinpkgFetcher.py  |   3 +-
 lib/_emerge/EbuildFetcher.py  |   3 +-
 lib/_emerge/EbuildPhase.py|  47 ++--
 lib/_emerge/SpawnProcess.py   |  58 +++---
 lib/portage/dbapi/_MergeProcess.py|   3 +-
 .../ebuild/_config/special_env_vars.py|   8 +-
 lib/portage/util/_async/BuildLogger.py| 109 ++
 lib/portage/util/_async/SchedulerInterface.py |  32 -
 man/make.conf.5   |   7 +-
 10 files changed, 243 insertions(+), 30 deletions(-)
 create mode 100644 lib/portage/util/_async/BuildLogger.py

diff --git a/lib/_emerge/AbstractEbuildProcess.py 
b/lib/_emerge/AbstractEbuildProcess.py
index 1c1955cfe..ae1aae55f 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2019 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -196,6 +196,7 @@ class AbstractEbuildProcess(SpawnProcess):
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
 
+   self.log_filter_file = 
self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
try:
SpawnProcess._start(self)
finally:
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 36d027de3..2e5861cc1 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -158,6 +158,7 @@ class _BinpkgFetcherProcess(SpawnProcess):
self.env = fetch_env
if settings.selinux_enabled():
self._selinux_type = settings["PORTAGE_FETCH_T"]
+   self.log_filter_file = 
settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
SpawnProcess._start(self)
 
def _pipe(self, fd_pipes):
diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py
index 1e40994fb..55349c33c 100644
--- a/lib/_emerge/EbuildFetcher.py
+++ b/lib/_emerge/EbuildFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import copy
@@ -225,6 +225,7 @@ class _EbuildFetcherProcess(ForkProcess):
settings["NOCOLOR"] = nocolor
 
self._settings = settings
+   self.log_filter_file = 
settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
ForkProcess._start(self)
 
# Free settings now since it's no longer needed in
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 477e0ba97..ddb3dc719 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -26,6 +26,8 @@ from portage.package.ebuild.prepare_build_dirs import 
(_prepare_workdir,
 from portage.util.futures.compat_coroutine import coroutine
 from portage.util import writemsg
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._async.BuildLogger import BuildLogger
+from portage.util.futures import asyncio
 from portage.util.futures.executor.fork import ForkExecutor
 
 try:
@@ -69,6 +71,11 @@ class EbuildPhase(CompositeTask):
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
 
def _start(self):
+   future = asyncio.ensure_future(self._async_start(), 
loop=self.scheduler)
+   self._start_task(AsyncTaskFuture(future=future), 
self._async_start_exit)
+
+   @coroutine
+   def _async_start(self):
 
need_builddir = self.phase not in 
EbuildProcess._phases_without_builddir
 
@@ -126,7 +133,7 @@ class EbuildPhase(CompositeTask):
# Force background=True for this header since it's 
intended
# for the log and it doesn

Re: [gentoo-portage-dev] [PATCH] _writer: fix unsafe finally clause (bug 728580)

2020-06-18 Thread Zac Medico
On 6/18/20 7:13 AM, Brian Dolbec wrote:
> On Thu, 18 Jun 2020 00:35:48 -0700
> Zac Medico  wrote:
> 
>> In the coroutine finally clause, do not call remove_writer in cases
>> where fd has been closed and then re-allocated to a concurrent
>> coroutine as in bug 716636.
>>
>> Also, assume that the caller will put the file in non-blocking mode
>> and close the file when done, so that this function is suitable for
>> use within a loop.
>>
>> Bug: https://bugs.gentoo.org/728580
>> Signed-off-by: Zac Medico 
>> ---
>>  lib/portage/util/futures/_asyncio/process.py | 11 -
>>  lib/portage/util/futures/_asyncio/streams.py | 50
>> ++-- 2 files changed, 33 insertions(+), 28
>> deletions(-)
>>
>> diff --git a/lib/portage/util/futures/_asyncio/process.py
>> b/lib/portage/util/futures/_asyncio/process.py index
>> 020164c9b..2d3e9b0fd 100644 ---
>> a/lib/portage/util/futures/_asyncio/process.py +++
>> b/lib/portage/util/futures/_asyncio/process.py @@ -1,9 +1,12 @@
>> -# Copyright 2018 Gentoo Foundation
>> +# Copyright 2018-2020 Gentoo Authors
>>  # Distributed under the terms of the GNU General Public License v2
>>  
>> +import os
>> +
>>  import portage
>>  portage.proxy.lazyimport.lazyimport(globals(),
>>  'portage.util.futures:asyncio',
>> +'portage.util.futures.unix_events:_set_nonblocking',
>>  )
>>  from portage.util.futures._asyncio.streams import _reader, _writer
>>  from portage.util.futures.compat_coroutine import coroutine,
>> coroutine_return @@ -59,7 +62,11 @@ class _Process(object):
>>  if input is not None:
>>  if self._proc.stdin is None:
>>  raise TypeError('communicate:
>> expected file or int, got {}'.format(type(self._proc.stdin)))
>> -writer =
>> asyncio.ensure_future(_writer(self._proc.stdin, input),
>> loop=self._loop)
>> +stdin = self._proc.stdin
>> +stdin = os.fdopen(stdin, 'wb', 0) if
>> isinstance(stdin, int) else stdin
>> +_set_nonblocking(stdin.fileno())
>> +writer =
>> asyncio.ensure_future(_writer(stdin, input, loop=self._loop),
>> loop=self._loop)
>> +writer.add_done_callback(lambda writer:
>> stdin.close()) 
>>  try:
>>  yield asyncio.wait(futures + [self.wait()],
>> loop=self._loop) diff --git
>> a/lib/portage/util/futures/_asyncio/streams.py
>> b/lib/portage/util/futures/_asyncio/streams.py index
>> 650a16491..870307e1e 100644 ---
>> a/lib/portage/util/futures/_asyncio/streams.py +++
>> b/lib/portage/util/futures/_asyncio/streams.py @@ -1,4 +1,4 @@ -#
>> Copyright 2018 Gentoo Foundation +# Copyright 2018-2020 Gentoo Authors
>>  # Distributed under the terms of the GNU General Public License v2
>>  
>>  import errno
>> @@ -8,7 +8,6 @@ import portage
>>  portage.proxy.lazyimport.lazyimport(globals(),
>>  '_emerge.PipeReader:PipeReader',
>>  'portage.util.futures:asyncio',
>> -'portage.util.futures.unix_events:_set_nonblocking',
>>  )
>>  from portage.util.futures.compat_coroutine import coroutine
>>  
>> @@ -59,38 +58,37 @@ class _Reader(object):
>>  @coroutine
>>  def _writer(output_file, content, loop=None):
>>  """
>> -Asynchronously write bytes to output file, and close it when
>> -done. If an EnvironmentError other than EAGAIN is
>> encountered,
>> -which typically indicates that the other end of the pipe has
>> -close, the error is raised. This function is a coroutine.
>> +Asynchronously write bytes to output file. The output file is
>> +assumed to be in non-blocking mode. If an EnvironmentError
>> +other than EAGAIN is encountered, which typically indicates
>> that
>> +the other end of the pipe has closed, the error is raised.
>> +This function is a coroutine.
>>  
>> -@param output_file: output file descriptor
>> -@type output_file: file or int
>> +@param output_file: output file
>> +@type output_file: file object
>>  @param content: content to write
>>  @type content: bytes
>>  @param loop: asyncio.AbstractEventLoop (or compatible)
>>  @type loop: event loop
>>  """
>> -fd = output_file if isinstance(output_file, int) else
>> output_file.fileno()
>> -_set_nonblocking(fd)
>>  loop = asyncio._wrap_loop(loop)
>> - 

[gentoo-portage-dev] [PATCH] _writer: fix unsafe finally clause (bug 728580)

2020-06-18 Thread Zac Medico
In the coroutine finally clause, do not call remove_writer in cases
where fd has been closed and then re-allocated to a concurrent
coroutine as in bug 716636.

Also, assume that the caller will put the file in non-blocking mode
and close the file when done, so that this function is suitable for
use within a loop.

Bug: https://bugs.gentoo.org/728580
Signed-off-by: Zac Medico 
---
 lib/portage/util/futures/_asyncio/process.py | 11 -
 lib/portage/util/futures/_asyncio/streams.py | 50 ++--
 2 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/process.py 
b/lib/portage/util/futures/_asyncio/process.py
index 020164c9b..2d3e9b0fd 100644
--- a/lib/portage/util/futures/_asyncio/process.py
+++ b/lib/portage/util/futures/_asyncio/process.py
@@ -1,9 +1,12 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+import os
+
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures:asyncio',
+   'portage.util.futures.unix_events:_set_nonblocking',
 )
 from portage.util.futures._asyncio.streams import _reader, _writer
 from portage.util.futures.compat_coroutine import coroutine, coroutine_return
@@ -59,7 +62,11 @@ class _Process(object):
if input is not None:
if self._proc.stdin is None:
raise TypeError('communicate: expected file or 
int, got {}'.format(type(self._proc.stdin)))
-   writer = 
asyncio.ensure_future(_writer(self._proc.stdin, input), loop=self._loop)
+   stdin = self._proc.stdin
+   stdin = os.fdopen(stdin, 'wb', 0) if isinstance(stdin, 
int) else stdin
+   _set_nonblocking(stdin.fileno())
+   writer = asyncio.ensure_future(_writer(stdin, input, 
loop=self._loop), loop=self._loop)
+   writer.add_done_callback(lambda writer: stdin.close())
 
try:
yield asyncio.wait(futures + [self.wait()], 
loop=self._loop)
diff --git a/lib/portage/util/futures/_asyncio/streams.py 
b/lib/portage/util/futures/_asyncio/streams.py
index 650a16491..870307e1e 100644
--- a/lib/portage/util/futures/_asyncio/streams.py
+++ b/lib/portage/util/futures/_asyncio/streams.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -8,7 +8,6 @@ import portage
 portage.proxy.lazyimport.lazyimport(globals(),
'_emerge.PipeReader:PipeReader',
'portage.util.futures:asyncio',
-   'portage.util.futures.unix_events:_set_nonblocking',
 )
 from portage.util.futures.compat_coroutine import coroutine
 
@@ -59,38 +58,37 @@ class _Reader(object):
 @coroutine
 def _writer(output_file, content, loop=None):
"""
-   Asynchronously write bytes to output file, and close it when
-   done. If an EnvironmentError other than EAGAIN is encountered,
-   which typically indicates that the other end of the pipe has
-   close, the error is raised. This function is a coroutine.
+   Asynchronously write bytes to output file. The output file is
+   assumed to be in non-blocking mode. If an EnvironmentError
+   other than EAGAIN is encountered, which typically indicates that
+   the other end of the pipe has closed, the error is raised.
+   This function is a coroutine.
 
-   @param output_file: output file descriptor
-   @type output_file: file or int
+   @param output_file: output file
+   @type output_file: file object
@param content: content to write
@type content: bytes
@param loop: asyncio.AbstractEventLoop (or compatible)
@type loop: event loop
"""
-   fd = output_file if isinstance(output_file, int) else 
output_file.fileno()
-   _set_nonblocking(fd)
loop = asyncio._wrap_loop(loop)
-   try:
-   while content:
+   fd = output_file.fileno()
+   while content:
+   try:
+   content = content[os.write(fd, content):]
+   except EnvironmentError as e:
+   if e.errno != errno.EAGAIN:
+   raise
waiter = loop.create_future()
-   loop.add_writer(fd, lambda: waiter.set_result(None))
+   loop.add_writer(fd, lambda: waiter.done() or 
waiter.set_result(None))
try:
yield waiter
-   while content:
-   try:
-   content = content[os.write(fd, 
content):]
- 

Re: [gentoo-portage-dev] erroneous behavior in 2-style USE dependencies?

2020-06-16 Thread Zac Medico
On 6/16/20 7:47 PM, Michael Lienhardt wrote:
> 
> 
> On 6/16/20 11:59 PM, Zac Medico wrote:
>> On 6/16/20 6:38 PM, Michael Lienhardt wrote:
>>> With the first version of DEPEND, emerge succeed:
>>> # emerge -pv app-misc/dummy-master
>>>
>>> These are the packages that would be merged, in order:
>>>
>>> Calculating dependencies... done!
>>> [ebuild  N ] app-misc/dummy-slave-2::gentoo  USE="-static-libs" 0 KiB
>>> [ebuild  N ] app-misc/dummy-master-1::gentoo  USE="-static-libs" 0 KiB
>>
>> This success is expected, yes? Do you suggest to change the behavior
>> somehow?
> 
> The way I interpret the PMS, this success is not expected:
>  the atom ">=app-misc/dummy-slave-1" matches the cpv "app-misc/dummy-slave-1" 
> which does not contains the use flag 'static-libs',
>  and thus I expected a 'missing use flag' error.

For this calculation, it would be a waste of time to read the IUSE
metadata for app-misc/dummy-slave-1, since app-misc/dummy-slave-2 is the
highest available version.

I hope that PMS does not specify that package managers must read IUSE
metadata for irrelevant package versions!

> I'm not suggesting to change the behavior of emerge, I'm saying that:
>  - the way I read the PMS, I expect behavior A, but in practice, I see 
> behavior B.
>  - what does the portage devs / PMS gurus think about that?
> - is my understanding of the PMS wrong, and it actually says "behavior B 
> is expected"?
> - if yes, where did I fail in my understanding?
> - if no, should emerge or the PMS be updated so they both describe the 
> same behavior?
>  - I will implement your ruling in my tool, which I try to match as closely 
> as possible to the PMS

In this context I think the spirit of what PMS says is that the package
manager should emit some kind of warning message if it finds missing
IUSE. Now, in the example above, if the package manager has no reason to
examine the IUSE metadata of app-misc/dummy-slave-1 because
app-misc/dummy-slave-2 is the highest available version, then there's no
opportunity for it to emit a warning message.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] erroneous behavior in 2-style USE dependencies?

2020-06-16 Thread Zac Medico
On 6/16/20 6:38 PM, Michael Lienhardt wrote:
> 
> 
> On 6/16/20 9:31 PM, Zac Medico wrote:
>> On 6/16/20 11:07 PM, Michael Lienhardt wrote:
>>> I'm sorry, my client didn't allow to send plain text email anymore...
>>>
>>> So, here is my original email.
>>>
>>> Dear all,
>>>
>>> My bad for not noticing it sooner, but when there is a dependency like 
>>> ">=sys-fs/udev-208-r1:0/0[static-libs?]" (that occurs in 
>>> virtual/libgudev-215-r3),
>>>  since 'static-libs' is not a use flags of sys-fs/udev-242, that cpv is 
>>> silently not considered during dependency solving by emerge.
>>> However, the PMS states:
>>>  - it is an error for a use dependency to be applied to an ebuild which 
>>> does not have the flag in question in IUSE_REFERENCEABLE
>>>  - For EAPIs listed in table 5.4 as not supporting profile defined IUSE 
>>> injection, IUSE_REFERENCEABLE is equal to the calculated IUSE value. For 
>>> EAPIs where profile defined IUSE injection is supported, IUSE_REFERENCEABLE 
>>> is equal to IUSE_EFFECTIVE
>>> And 'static-libs' is not in the IUSE_EFFECTIVE of sys-fs/udev-242 (that 
>>> ebuild has EAPI=6).
>>> So it seems to me that this current behavior of emerge should be considered 
>>> an error, no? Or the PMS should be updated?

Yes, according to PMS it's a "error" but emerge has never reported it as
such. Adding an error message to emerge sounds fine to me.

>>> This is related to the tool I'm working on: should my tool allow this 
>>> behavior, or fail like it is currently doing (I guess the former)?
> 
>> It's valid as a 4-style dependency with use-dep-defaults.

Actually, it doesn't have use-dep-defaults but somehow my brain
associated it with that because you were talking about IUSE.

> I know. Except that in virtual/libgudev-215-r3.ebuild we have in the DEPEND:
>  
> >=sys-fs/udev-208-r1:0/0[${MULTILIB_USEDEP},gudev(-),introspection(-)?,static-libs?]
> It is a 2-style dependency (following the PMS).
> 
> I checked with 3 dummy packages
> 
> 1. app-misc/dummy-master-1
> EAPI=6
> SLOT=0
> KEYWORDS="amd64"
> IUSE="static-libs"
> DEPEND=">=app-misc/dummy-slave-1[static-libs?]"
> #DEPEND="=app-misc/dummy-slave-1[static-libs?]"
> 
> 2. app-misc/dummy-slave-1
> EAPI=6
> SLOT=0
> KEYWORDS="amd64"
> IUSE=""
> DEPEND=""
> 
> 3. app-misc/dummy-slave-2
> EAPI=6
> SLOT=0
> KEYWORDS="amd64"
> IUSE="static-libs"
> DEPEND=""
> 
> With the first version of DEPEND, emerge succeed:
> # emerge -pv app-misc/dummy-master
> 
> These are the packages that would be merged, in order:
> 
> Calculating dependencies... done!
> [ebuild  N ] app-misc/dummy-slave-2::gentoo  USE="-static-libs" 0 KiB
> [ebuild  N ] app-misc/dummy-master-1::gentoo  USE="-static-libs" 0 KiB

This success is expected, yes? Do you suggest to change the behavior
somehow?

> With the second version of DEPEND, emerge fails:
> # emerge -pv app-misc/dummy-master
> 
> These are the packages that would be merged, in order:
> 
> Calculating dependencies... done!
> 
> emerge: there are no ebuilds built with USE flags to satisfy 
> "=app-misc/dummy-slave-1[static-libs?]".
> !!! One of the following packages is required to complete your request:
> - app-misc/dummy-slave-1::gentoo (Missing IUSE: static-libs)
> (dependency required by "app-misc/dummy-master-1::gentoo" [ebuild])
> (dependency required by "app-misc/dummy-master" [argument])

This failure is expected, yes? Do you suggest to change the behavior
somehow?
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] erroneous behavior in 2-style USE dependencies?

2020-06-16 Thread Zac Medico
On 6/16/20 11:43 AM, michael.lienhardt wrote:
> ‌Dear all,
> 
> My bad for not noticing it sooner, but when there is a dependency like
> ">=sys-fs/udev-208-r1:0/0[static-libs?]" (that occurs in
> virtual/libgudev-215-r3), since 'static-libs' is not a use flags of
> sys-fs/udev-242, it is silently not considered during dependency solving
> by emerge.
> However, the PMS states:
>  - it is an error for a use dependency to be applied to an ebuild which
> does not have the flag in question in IUSE_REFERENCEABLE
>  - For EAPIs listed in table 5.4
>  as not
> supporting profile defined IUSE injection, IUSE_REFERENCEABLE is equal
> to the calculated IUSE value. For EAPIs where profile defined IUSE
> injection is supported, IUSE_REFERENCEABLE is equal to IUSE_EFFECTIVE
> And 'static-libs' is not in the IUSE_EFFECTIVE of sys-fs/udev-242 (that
> ebuild has EAPI=6).
> So it seems to me that this current behavior of emerge should be
> considered an error, no? Or the PMS should be updated?

It's valid as a 4-style dependency with use-dep-defaults.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH] Use env to find python

2020-06-16 Thread Zac Medico
On 6/16/20 10:46 AM, Mike Gilbert wrote:
> On Tue, Jun 16, 2020 at 1:45 PM Mike Gilbert  wrote:
>>
>> On Mon, Jun 15, 2020 at 9:39 AM Sid Spry  wrote:
>>>
>>> On Mon, Jun 15, 2020, at 2:36 AM, Ulrich Mueller wrote:
 But we know that it is in /usr/bin, so why add yet another indirection?

 Attachments:
 * signature.asc
>>>
>>> Ah, sorry -- I forgot to note this here. If you wish to support prefix it 
>>> is possible it may not be in /usr/bin. Granted I am not sure if the prefix 
>>> stage3 I was using is old enough to be broken in some way, but adding this 
>>> would prevent future breakage.
>>
>> The portage ebuild and the python distutils module already take care
>> of updating shebangs at install time.
> 
> I suppose your patch might be useful if you are trying to run portage
> from a git checkout on a prefix system.
> 

So, given that the ebuild updates shebangs automatically, should't we
optimize the default shebangs to be as flexible as possible?
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


Re: [gentoo-portage-dev] [PATCH 1/1] repoman: deprecate netsurf.eclass.

2020-06-16 Thread Zac Medico
On 6/16/20 10:36 AM, Michael Orlitzky wrote:
> While investigating bug 489542, it became clear that the netsurf
> eclass is deprecated in spirit if not in fact. All of the newer
> netsurf ebuilds use a custom function loaded from a script that is
> shipped in netsurf-buildsystem's $FILESDIR to do (some of) what the
> eclass used to do. That function probably does belong in an eclass,
> but at this point, we should throw this thing out and start from
> scratch.
> 
> By deprecating the eclass, we make sure that no one else inherits it
> during the time it takes to purge the two remaining consumers. Then,
> once those are gone, the build system script magic can be put back
> into an eclass, and its consumers updated one-at-a-time.
> 
> Bug: https://bugs.gentoo.org/489542
> Bug: https://bugs.gentoo.org/637824
> Signed-off-by: Michael Orlitzky 
> ---
>  repoman/lib/repoman/modules/linechecks/deprecated/inherit.py | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py 
> b/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py
> index 5848a0c37..60410347b 100644
> --- a/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py
> +++ b/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py
> @@ -33,6 +33,7 @@ class InheritDeprecated(LineCheck):
>   "gst-plugins10": "gstreamer",
>   "ltprune": False,
>   "mono": "mono-env",
> + "netsurf": False,
>   "python": "python-r1 / python-single-r1 / python-any-r1",
>   "ruby": "ruby-ng",
>   "user": "GLEP 81",
> 

Thanks, applied:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=a73024729860f9224b8d1660d24c450080b67d9f
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] [PATCH] Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)

2020-06-13 Thread Zac Medico
This variable specifies a command that filters build log output to a
log file. The plan is to extend this to support a separate filter for
tty output in the future.

In order to enable the EbuildPhase class to write elog messages to
the build log with PORTAGE_LOG_FILTER_FILE_CMD support, convert its
_elog method to a coroutine, and add a SchedulerInterface async_output
method for it to use.

Use a new BuildLogger class to manage log output (with or without a
filter command), with compression support provided by PipeLogger.
BuildLogger has a stdin property which provides access to a writable
binary file stream (refers to a pipe) that log content is written to.

Bug: https://bugs.gentoo.org/709746
Signed-off-by: Zac Medico 
---
 lib/_emerge/AbstractEbuildProcess.py  |   2 +
 lib/_emerge/EbuildPhase.py|  47 ++--
 lib/_emerge/SpawnProcess.py   |  56 ++---
 .../ebuild/_config/special_env_vars.py|   8 +-
 lib/portage/util/_async/BuildLogger.py| 109 ++
 lib/portage/util/_async/SchedulerInterface.py |  32 -
 man/make.conf.5   |   7 +-
 7 files changed, 236 insertions(+), 25 deletions(-)
 create mode 100644 lib/portage/util/_async/BuildLogger.py

diff --git a/lib/_emerge/AbstractEbuildProcess.py 
b/lib/_emerge/AbstractEbuildProcess.py
index 1c1955cfe..5fd4e3a1e 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -196,6 +196,8 @@ class AbstractEbuildProcess(SpawnProcess):
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
 
+   self.log_filter_file = 
self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
+
try:
SpawnProcess._start(self)
finally:
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 477e0ba97..ddb3dc719 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -26,6 +26,8 @@ from portage.package.ebuild.prepare_build_dirs import 
(_prepare_workdir,
 from portage.util.futures.compat_coroutine import coroutine
 from portage.util import writemsg
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._async.BuildLogger import BuildLogger
+from portage.util.futures import asyncio
 from portage.util.futures.executor.fork import ForkExecutor
 
 try:
@@ -69,6 +71,11 @@ class EbuildPhase(CompositeTask):
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
 
def _start(self):
+   future = asyncio.ensure_future(self._async_start(), 
loop=self.scheduler)
+   self._start_task(AsyncTaskFuture(future=future), 
self._async_start_exit)
+
+   @coroutine
+   def _async_start(self):
 
need_builddir = self.phase not in 
EbuildProcess._phases_without_builddir
 
@@ -126,7 +133,7 @@ class EbuildPhase(CompositeTask):
# Force background=True for this header since it's 
intended
# for the log and it doesn't necessarily need to be 
visible
# elsewhere.
-   self._elog('einfo', msg, background=True)
+   yield self._elog('einfo', msg, background=True)
 
if self.phase == 'package':
if 'PORTAGE_BINPKG_TMPFILE' not in self.settings:
@@ -134,6 +141,12 @@ class EbuildPhase(CompositeTask):
os.path.join(self.settings['PKGDIR'],
self.settings['CATEGORY'], 
self.settings['PF']) + '.tbz2'
 
+   def _async_start_exit(self, task):
+   task.future.cancelled() or task.future.result()
+   if self._default_exit(task) != os.EX_OK:
+   self.wait()
+   return
+
if self.phase in ("pretend", "prerm"):
env_extractor = 
BinpkgEnvExtractor(background=self.background,
scheduler=self.scheduler, 
settings=self.settings)
@@ -391,6 +404,7 @@ class EbuildPhase(CompositeTask):
self.returncode = 1
self.wait()
 
+   @coroutine
def _elog(self, elog_funcname, lines, background=None):
if background is None:
background = self.background
@@ -407,11 +421,30 @@ class EbuildPhase(CompositeTask):
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
-   log_path = None
-   if self.settings.get("PORTAGE_BACKGROUND") != 
"subprocess":
-   log_path = self.settings.get("PORTAGE_LOG_FILE")
-   self.scheduler.output(msg, log_path=log_path

Re: [gentoo-portage-dev] [PATCH] repos.conf: add bool sync-openpgp-key-refresh option (bug 661518)

2020-06-12 Thread Zac Medico
On 6/12/20 6:48 PM, Brian Dolbec wrote:
> On Fri, 12 Jun 2020 16:51:51 -0700
> Zac Medico  wrote:
> 
>> Add a sync-openpgp-key-refresh option that makes it possible to
>> disable key refresh, which may be useful in cases when it is not
>> possible to refresh keys.
>>
>> Key refresh is enabled by default, and if it is disabled then
>> the SyncBase._refresh_keys method will output an ewarn message
>> like this when the --quiet option is not enabled:
>>
>>  * Key refresh is disabled via a repos.conf sync-openpgp-key-refresh
>>  * setting, and this is a security vulnerability because it prevents
>>  * detection of revoked keys!
>>
>> Bug: https://bugs.gentoo.org/661518
>> Signed-off-by: Zac Medico 
>> ---
>>  lib/portage/repository/config.py | 10 +-
>>  lib/portage/sync/syncbase.py |  9 -
>>  man/portage.5|  9 -
>>  3 files changed, 25 insertions(+), 3 deletions(-)
>>
>> diff --git a/lib/portage/repository/config.py
>> b/lib/portage/repository/config.py index 50ab18026..6155c130a 100644
>> --- a/lib/portage/repository/config.py
>> +++ b/lib/portage/repository/config.py
>> @@ -1,4 +1,4 @@
>> -# Copyright 2010-2019 Gentoo Authors
>> +# Copyright 2010-2020 Gentoo Authors
>>  # Distributed under the terms of the GNU General Public License v2
>>  
>>  from __future__ import unicode_literals
>> @@ -113,6 +113,7 @@ class RepoConfig(object):
>>  'sync_hooks_only_on_change',
>>  'sync_openpgp_keyserver',
>>  'sync_openpgp_key_path',
>> +'sync_openpgp_key_refresh',
>>  'sync_openpgp_key_refresh_retry_count',
>>  'sync_openpgp_key_refresh_retry_delay_exp_base',
>>  'sync_openpgp_key_refresh_retry_delay_max',
>> @@ -233,6 +234,9 @@ class RepoConfig(object):
>>  self.sync_openpgp_key_path = repo_opts.get(
>>  'sync-openpgp-key-path', None)
>>  
>> +self.sync_openpgp_key_refresh = repo_opts.get(
>> +'sync-openpgp-key-refresh', 'true').lower()
>> in ('true', 'yes') +
>>  for k in ('sync_openpgp_key_refresh_retry_count',
>>  'sync_openpgp_key_refresh_retry_delay_exp_base',
>>  'sync_openpgp_key_refresh_retry_delay_max',
>> @@ -497,6 +501,8 @@ class RepoConfig(object):
>>  repo_msg.append(indent + "location: " +
>> self.location) if not self.strict_misc_digests:
>>  repo_msg.append(indent +
>> "strict-misc-digests: false")
>> +if not self.sync_openpgp_key_refresh:
>> +repo_msg.append(indent +
>> "sync-openpgp-key-refresh: no") if self.sync_type:
>>  repo_msg.append(indent + "sync-type: " +
>> self.sync_type) if self.sync_umask:
>> @@ -609,6 +615,7 @@ class RepoConfigLoader(object):
>>  
>> 'sync_hooks_only_on_change',
>>  
>> 'sync_openpgp_keyserver',
>>  'sync_openpgp_key_path',
>> +
>> 'sync_openpgp_key_refresh', 'sync_openpgp_key_refresh_retry_count',
>>  
>> 'sync_openpgp_key_refresh_retry_delay_exp_base',
>>  
>> 'sync_openpgp_key_refresh_retry_delay_max',
>> @@ -1047,6 +1054,7 @@ class RepoConfigLoader(object):
>>  bool_keys = (
>>  "strict_misc_digests",
>>  "sync_allow_hardlinks",
>> +"sync_openpgp_key_refresh",
>>  "sync_rcu",
>>  )
>>  str_or_int_keys = (
>> diff --git a/lib/portage/sync/syncbase.py
>> b/lib/portage/sync/syncbase.py index 46644d68e..74818a420 100644
>> --- a/lib/portage/sync/syncbase.py
>> +++ b/lib/portage/sync/syncbase.py
>> @@ -1,4 +1,4 @@
>> -# Copyright 2014-2018 Gentoo Foundation
>> +# Copyright 2014-2020 Gentoo Authors
>>  # Distributed under the terms of the GNU General Public License v2
>>  
>>  '''
>> @@ -252,6 +252,13 @@ class SyncBase(object):
>>  @type openpgp_env: gemato.openpgp.OpenPGPEnvironment
>>  """
>>  out = portage.output.EOutput(quiet=('--quiet' in
>> self.options['emerge_config'].opt

  1   2   3   4   5   6   7   8   9   10   >