subject:"\[gentoo\-commits\] proj\/portage\:master commit in\: pym\/portage\/"

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/

2018-07-17 Thread Zac Medico

commit: a7c7af98d755f34e84d1f0f847e2c0d5cc5b7e2f
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Jul 18 03:36:59 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jul 18 03:40:05 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7c7af98

EventLoop: raise TypeError for unexpected call_* keyword args

 pym/portage/util/_eventloop/EventLoop.py | 24 
 1 file changed, 24 insertions(+)

diff --git a/pym/portage/util/_eventloop/EventLoop.py 
b/pym/portage/util/_eventloop/EventLoop.py
index 084ff0c18..ffd12cff9 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -859,11 +859,23 @@ class EventLoop(object):
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
+   try:
+   unexpected = next(key for key in kwargs if key != 
'context')
+   except StopIteration:
+   pass
+   else:
+   raise TypeError("call_soon() got an unexpected keyword 
argument '%s'" % unexpected)
return self._handle(self._idle_add(
self._call_soon_callback(callback, args)), self)
 
def call_soon_threadsafe(self, callback, *args, **kwargs):
"""Like call_soon(), but thread safe."""
+   try:
+   unexpected = next(key for key in kwargs if key != 
'context')
+   except StopIteration:
+   pass
+   else:
+   raise TypeError("call_soon_threadsafe() got an 
unexpected keyword argument '%s'" % unexpected)
# idle_add provides thread safety
return self._handle(self.idle_add(
self._call_soon_callback(callback, args)), self)
@@ -909,6 +921,12 @@ class EventLoop(object):
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
+   try:
+   unexpected = next(key for key in kwargs if key != 
'context')
+   except StopIteration:
+   pass
+   else:
+   raise TypeError("call_later() got an unexpected keyword 
argument '%s'" % unexpected)
return self._handle(self.timeout_add(
delay * 1000, self._call_soon_callback(callback, 
args)), self)
 
@@ -936,6 +954,12 @@ class EventLoop(object):
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
+   try:
+   unexpected = next(key for key in kwargs if key != 
'context')
+   except StopIteration:
+   pass
+   else:
+   raise TypeError("call_at() got an unexpected keyword 
argument '%s'" % unexpected)
delta = when - self.time()
return self.call_later(delta if delta > 0 else 0, callback, 
*args)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/

2018-07-17 Thread Zac Medico

commit: e46dd735cd4dde58cf3f8ef3cd2b8b29561f5b3e
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Jul 17 19:27:28 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Jul 17 19:27:28 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e46dd735

EventLoop: use python2.7 compatible **kwargs for call_* context arg

Since python2.7 does not allow positional default arguments after
*args, use **kwargs instead.

Fixes: ae8cc32ccd81 ("EventLoop: add call_* context arg for python3.7 compat")

 pym/portage/util/_eventloop/EventLoop.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/util/_eventloop/EventLoop.py 
b/pym/portage/util/_eventloop/EventLoop.py
index 69ccbac2c..084ff0c18 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -832,7 +832,7 @@ class EventLoop(object):
 
return future.result()
 
-   def call_soon(self, callback, *args, context=None):
+   def call_soon(self, callback, *args, **kwargs):
"""
Arrange for a callback to be called as soon as possible. The 
callback
is called after call_soon() returns, when control returns to 
the event
@@ -862,7 +862,7 @@ class EventLoop(object):
return self._handle(self._idle_add(
self._call_soon_callback(callback, args)), self)
 
-   def call_soon_threadsafe(self, callback, *args, context=None):
+   def call_soon_threadsafe(self, callback, *args, **kwargs):
"""Like call_soon(), but thread safe."""
# idle_add provides thread safety
return self._handle(self.idle_add(
@@ -877,7 +877,7 @@ class EventLoop(object):
"""
return monotonic()
 
-   def call_later(self, delay, callback, *args, context=None):
+   def call_later(self, delay, callback, *args, **kwargs):
"""
Arrange for the callback to be called after the given delay 
seconds
(either an int or float).
@@ -912,7 +912,7 @@ class EventLoop(object):
return self._handle(self.timeout_add(
delay * 1000, self._call_soon_callback(callback, 
args)), self)
 
-   def call_at(self, when, callback, *args, context=None):
+   def call_at(self, when, callback, *args, **kwargs):
"""
Arrange for the callback to be called at the given absolute
timestamp when (an int or float), using the same time reference 
as

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/

2018-07-17 Thread Zac Medico

commit: ae8cc32ccd812661650647feffa1b10fc3ab5837
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Jul 17 19:04:28 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Jul 17 19:05:38 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ae8cc32c

EventLoop: add call_* context arg for python3.7 compat

The context argument currently does nothing, but exists for minimal
interoperability with Future instances that require it for PEP 567.

 pym/portage/util/_eventloop/EventLoop.py | 26 ++
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/pym/portage/util/_eventloop/EventLoop.py 
b/pym/portage/util/_eventloop/EventLoop.py
index df76374d9..69ccbac2c 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -832,7 +832,7 @@ class EventLoop(object):
 
return future.result()
 
-   def call_soon(self, callback, *args):
+   def call_soon(self, callback, *args, context=None):
"""
Arrange for a callback to be called as soon as possible. The 
callback
is called after call_soon() returns, when control returns to 
the event
@@ -844,18 +844,25 @@ class EventLoop(object):
Any positional arguments after the callback will be passed to 
the
callback when it is called.
 
+   The context argument currently does nothing, but exists for 
minimal
+   interoperability with Future instances that require it for PEP 
567.
+
An object compatible with asyncio.Handle is returned, which can
be used to cancel the callback.
 
@type callback: callable
@param callback: a function to call
+   @type context: contextvars.Context
+   @param context: An optional keyword-only context argument allows
+   specifying a custom contextvars.Context for the 
callback to run
+   in. The current context is used when no context is 
provided.
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
return self._handle(self._idle_add(
self._call_soon_callback(callback, args)), self)
 
-   def call_soon_threadsafe(self, callback, *args):
+   def call_soon_threadsafe(self, callback, *args, context=None):
"""Like call_soon(), but thread safe."""
# idle_add provides thread safety
return self._handle(self.idle_add(
@@ -870,7 +877,7 @@ class EventLoop(object):
"""
return monotonic()
 
-   def call_later(self, delay, callback, *args):
+   def call_later(self, delay, callback, *args, context=None):
"""
Arrange for the callback to be called after the given delay 
seconds
(either an int or float).
@@ -886,19 +893,26 @@ class EventLoop(object):
it is called. If you want the callback to be called with some 
named
arguments, use a closure or functools.partial().
 
+   The context argument currently does nothing, but exists for 
minimal
+   interoperability with Future instances that require it for PEP 
567.
+
Use functools.partial to pass keywords to the callback.
 
@type delay: int or float
@param delay: delay seconds
@type callback: callable
@param callback: a function to call
+   @type context: contextvars.Context
+   @param context: An optional keyword-only context argument allows
+   specifying a custom contextvars.Context for the 
callback to run
+   in. The current context is used when no context is 
provided.
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
return self._handle(self.timeout_add(
delay * 1000, self._call_soon_callback(callback, 
args)), self)
 
-   def call_at(self, when, callback, *args):
+   def call_at(self, when, callback, *args, context=None):
"""
Arrange for the callback to be called at the given absolute
timestamp when (an int or float), using the same time reference 
as
@@ -915,6 +929,10 @@ class EventLoop(object):
@param when: absolute timestamp when to call callback
@type callback: callable
@param callback: a function to call
+   @type context: contextvars.Context
+   @param context: An optional keyword-only context argument allows
+   specifying a custom contextvars.Context for the 
callback to run
+

[gentoo-commits] proj/portage:master commit in: pym/portage/repository/

2018-07-15 Thread Zac Medico

commit: 01f49cb12c66026e5b225912f9759b50ff2e2289
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jul 15 20:55:48 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Jul 16 05:53:05 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=01f49cb1

RepoConfig: fix make.conf PORTDIR override (bug 661276)

Pass the main-repo name from repos.conf as the repository name
for the PORTDIR RepoConfig constructor, so that it can override
the main repo location even if the profiles/repo_name file does
not exist yet (like in a stage3 tarball).

Bug: https://bugs.gentoo.org/661276
Reported-by: Jorge Manuel B. S. Vicetto  gentoo.org>

 pym/portage/repository/config.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index bf2b6dd03..e45e67dec 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -524,7 +524,8 @@ class RepoConfigLoader(object):
(base_priority == 0 and ov is portdir):
repo_opts = default_repo_opts.copy()
repo_opts['location'] = ov
-   repo = RepoConfig(None, repo_opts, 
local_config=local_config)
+   name = prepos['DEFAULT'].main_repo if 
ov is portdir else None
+   repo = RepoConfig(name, repo_opts, 
local_config=local_config)
# repos_conf_opts contains options from 
repos.conf
repos_conf_opts = 
repos_conf.get(repo.name)
if repos_conf_opts is not None:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

2018-07-15 Thread Zac Medico

commit: 84431fe188ed1b3e2e7b52918329bd696d8304b1
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Jul 16 05:07:10 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Jul 16 05:18:51 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=84431fe1

portdbapi.cp_list: honor porttrees modifications (bug 650814)

Consumers, such as repoman and emirrordist, may modify the porttrees
attribute in order to modify the effective set of repositories for all
portdbapi operations.

Fixes: 27eeeb6e4fc8 ("portdbapi.cp_list: cache repo associations (bug 650814)")
Bug: https://bugs.gentoo.org/650814

 pym/portage/dbapi/porttree.py | 25 +++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 2e271ea76..677452273 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -322,6 +322,26 @@ class portdbapi(dbapi):
self._better_cache = None
self._broken_ebuilds = set()
 
+   def _set_porttrees(self, porttrees):
+   """
+   Consumers, such as repoman and emirrordist, may modify the 
porttrees
+   attribute in order to modify the effective set of repositories 
for
+   all portdbapi operations.
+
+   @param porttrees: list of repo locations, in ascending order by
+   repo priority
+   @type porttrees: list
+   """
+   self._porttrees_repos = portage.OrderedDict((repo.name, repo)
+   for repo in 
(self.repositories.get_repo_for_location(location)
+   for location in porttrees))
+   self._porttrees = tuple(porttrees)
+
+   def _get_porttrees(self):
+   return self._porttrees
+
+   porttrees = property(_get_porttrees, _set_porttrees)
+
@property
def _event_loop(self):
if portage._internal_caller:
@@ -972,9 +992,10 @@ class portdbapi(dbapi):
repos = 
[self.repositories.get_repo_for_location(location)
for location in mytree]
elif self._better_cache is None:
-   repos = list(self.repositories)
+   repos = self._porttrees_repos.values()
else:
-   repos = reversed(self._better_cache[mycp])
+   repos = [repo for repo in 
reversed(self._better_cache[mycp])
+   if repo.name in self._porttrees_repos]
mylist = []
for repo in repos:
oroot = repo.location

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/

2018-07-15 Thread Zac Medico

commit: 2de6ba26fd45fe942329ac928029e4edf603e125
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jul 15 04:54:45 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jul 15 21:47:33 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2de6ba26

config: cache profile.bashrc stat results (bug 649806)

Optimize config to stat profile.bashrc files once in the constructor,
in order to avoid repeated stat calls in the setcpv method.

Bug: https://bugs.gentoo.org/649806
Reviewed-by: Brian Dolbec  gentoo.org>

 pym/portage/package/ebuild/config.py | 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/pym/portage/package/ebuild/config.py 
b/pym/portage/package/ebuild/config.py
index 88acac5cc..320d9f6c0 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -274,6 +274,7 @@ class config(object):
self.mycpv = clone.mycpv
self._setcpv_args_hash = clone._setcpv_args_hash
self._soname_provided = clone._soname_provided
+   self._profile_bashrc = clone._profile_bashrc
 
# immutable attributes (internal policy ensures lack of 
mutation)
self._locations_manager = clone._locations_manager
@@ -725,6 +726,10 @@ class config(object):
self._license_manager.extract_global_changes( \

self.configdict["conf"].get("ACCEPT_LICENSE", ""))
 
+   # profile.bashrc
+   self._profile_bashrc = 
tuple(os.path.isfile(os.path.join(profile.location, 'profile.bashrc'))
+   for profile in profiles_complex)
+
if local_config:
#package.properties
propdict = grabdict_package(os.path.join(
@@ -1596,11 +1601,9 @@ class config(object):
 
bashrc_files = []
 
-   for profile in self._locations_manager.profiles_complex:
-   profile_bashrc = os.path.join(profile.location,
-   'profile.bashrc')
-   if os.path.exists(profile_bashrc):
-   bashrc_files.append(profile_bashrc)
+   for profile, profile_bashrc in 
zip(self._locations_manager.profiles_complex, self._profile_bashrc):
+   if profile_bashrc:
+   
bashrc_files.append(os.path.join(profile.location, 'profile.bashrc'))
if profile in self._pbashrcdict:
cpdict = self._pbashrcdict[profile].get(cp)
if cpdict:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

2018-07-15 Thread Zac Medico

commit: 31edfe62541d8b55c64584b056ce64ad759f6a8e
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jul 15 21:38:30 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jul 15 21:38:30 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=31edfe62

portdbapi.xmatch: return separate items for each repo

Since xmatch match-all shares cache with the cp_list method, it needs
to return separate items when the same package version is found in
multiple repos.

Fixes: 27eeeb6e4fc8 ("portdbapi.cp_list: cache repo associations (bug 650814)")
Bug: https://bugs.gentoo.org/650814

 pym/portage/dbapi/porttree.py | 19 +++
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index f6076ee2b..2e271ea76 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -1108,20 +1108,10 @@ class portdbapi(dbapi):
else:
iterfunc = iter
 
-   if mydep.repo is not None:
-   repos = [mydep.repo]
-   else:
-   # We iterate over self.porttrees, since it's 
common to
-   # tweak this attribute in order to adjust match 
behavior.
-   repos = []
-   for tree in reversed(self.porttrees):
-   
repos.append(self.repositories.get_name_for_location(tree))
-
for cpv in iterfunc(mylist):
-   for repo in repos:
try:
metadata = dict(zip(aux_keys,
-   self.aux_get(cpv, 
aux_keys, myrepo=repo)))
+   self.aux_get(cpv, 
aux_keys, myrepo=cpv.repo)))
except KeyError:
# ebuild not in this repo, or 
masked by corruption
continue
@@ -1145,11 +1135,8 @@ class portdbapi(dbapi):
continue
 
myval.append(pkg_str)
-   # only yield a given cpv once
-   break
-
-   if myval and single_match:
-   break
+   if single_match:
+   break
 
if single_match:
if myval:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

2018-07-14 Thread Zac Medico

commit: 3a25c3fa13d7c62ba8c00d6c7a75eb907d34b568
Author: Zac Medico  gentoo  org>
AuthorDate: Fri Jul 13 03:58:04 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Jul 14 23:54:29 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3a25c3fa

_unmerge_dirs: revisit parents of removed symlinks (bug 640058)

When removal of a symlink is triggered by removal of the directory
that it points to, revisit the parent directories of the symlink.

Bug: https://bugs.gentoo.org/640058
Reviewed-by: Brian Dolbec  gentoo.org>

 pym/portage/dbapi/vartree.py | 25 +++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 1a86940f1..a104306eb 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2753,9 +2753,13 @@ class dblink(object):
real_root = self.settings['ROOT']
 
dirs = sorted(dirs)
-   dirs.reverse()
+   revisit = {}
 
-   for obj, inode_key in dirs:
+   while True:
+   try:
+   obj, inode_key = dirs.pop()
+   except IndexError:
+   break
# Treat any directory named "info" as a candidate here,
# since it might have been in INFOPATH previously even
# though it may not be there now.
@@ -2818,6 +2822,7 @@ class dblink(object):
raise
if e.errno != errno.ENOENT:
show_unmerge("---", 
unmerge_desc["!empty"], "dir", obj)
+   revisit[obj] = inode_key
 
# Since we didn't remove this directory, record 
the directory
# itself for use in syncfs calls, if we have 
removed another
@@ -2838,6 +2843,7 @@ class dblink(object):
# no need to protect symlinks that point to it.
unmerge_syms = 
protected_symlinks.pop(inode_key, None)
if unmerge_syms is not None:
+   parents = []
for relative_path in unmerge_syms:
obj = os.path.join(real_root,

relative_path.lstrip(os.sep))
@@ -2849,6 +2855,21 @@ class dblink(object):
raise
del e
show_unmerge("!!!", "", 
"sym", obj)
+   else:
+   
parents.append(os.path.dirname(obj))
+
+   if parents:
+   # Revisit parents recursively 
(bug 640058).
+   recursive_parents = []
+   for parent in set(parents):
+   while parent in revisit:
+   
recursive_parents.append(parent)
+   parent = 
os.path.dirname(parent)
+   if parent == 
'/':
+   break
+
+   for parent in 
sorted(set(recursive_parents)):
+   dirs.append((parent, 
revisit.pop(parent)))
 
def isowner(self, filename, destroot=None):
"""

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/, pym/portage/util/futures/

2018-07-11 Thread Zac Medico

commit: edba848a013e1797faeacc1911a5e6571fca3ca7
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Jul  5 04:44:42 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jul 11 07:40:59 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=edba848a

Add python2 compatible coroutine support (bug 660426)

For readability, it's desirable to make asynchronous code use
coroutines to avoid callbacks when possible. For python2 compatibility,
generators that yield Futures can be used to implement coroutines.

Add a compat_coroutine module which provides a @coroutine decorator
and a coroutine_return function that can be used to return a value
from a generator. The decorated function returns a Future which is
done when the generator is exhausted. Usage is very similar to asyncio
coroutine usage in python3.4 (see unit tests).

Bug: https://bugs.gentoo.org/660426

 .../tests/util/futures/test_compat_coroutine.py| 159 +
 pym/portage/util/futures/compat_coroutine.py   | 112 +++
 2 files changed, 271 insertions(+)

diff --git a/pym/portage/tests/util/futures/test_compat_coroutine.py 
b/pym/portage/tests/util/futures/test_compat_coroutine.py
new file mode 100644
index 0..cbc070869
--- /dev/null
+++ b/pym/portage/tests/util/futures/test_compat_coroutine.py
@@ -0,0 +1,159 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import (
+   coroutine,
+   coroutine_return,
+)
+from portage.tests import TestCase
+
+
+class CompatCoroutineTestCase(TestCase):
+
+   def test_returning_coroutine(self):
+   @coroutine
+   def returning_coroutine():
+   yield asyncio.sleep(0)
+   coroutine_return('success')
+
+   self.assertEqual('success',
+   
asyncio.get_event_loop().run_until_complete(returning_coroutine()))
+
+   def test_raising_coroutine(self):
+
+   class TestException(Exception):
+   pass
+
+   @coroutine
+   def raising_coroutine():
+   yield asyncio.sleep(0)
+   raise TestException('exception')
+
+   self.assertRaises(TestException,
+   asyncio.get_event_loop().run_until_complete, 
raising_coroutine())
+
+   def test_catching_coroutine(self):
+
+   class TestException(Exception):
+   pass
+
+   @coroutine
+   def catching_coroutine(loop=None):
+   loop = asyncio._wrap_loop(loop)
+   future = loop.create_future()
+   loop.call_soon(future.set_exception, 
TestException('exception'))
+   try:
+   yield future
+   except TestException:
+   self.assertTrue(True)
+   else:
+   self.assertTrue(False)
+   coroutine_return('success')
+
+   loop = asyncio.get_event_loop()
+   self.assertEqual('success',
+   loop.run_until_complete(catching_coroutine(loop=loop)))
+
+   def test_cancelled_coroutine(self):
+
+   @coroutine
+   def cancelled_coroutine(loop=None):
+   loop = asyncio._wrap_loop(loop)
+   while True:
+   yield loop.create_future()
+
+   loop = asyncio.get_event_loop()
+   future = cancelled_coroutine(loop=loop)
+   loop.call_soon(future.cancel)
+
+   self.assertRaises(asyncio.CancelledError,
+   loop.run_until_complete, future)
+
+   def test_cancelled_future(self):
+
+   @coroutine
+   def cancelled_future_coroutine(loop=None):
+   loop = asyncio._wrap_loop(loop)
+   while True:
+   future = loop.create_future()
+   loop.call_soon(future.cancel)
+   yield future
+
+   loop = asyncio.get_event_loop()
+   self.assertRaises(asyncio.CancelledError,
+   loop.run_until_complete, 
cancelled_future_coroutine(loop=loop))
+
+   def test_yield_expression_result(self):
+   @coroutine
+   def yield_expression_coroutine():
+   for i in range(3):
+   x = yield asyncio.sleep(0, result=i)
+   self.assertEqual(x, i)
+
+   
asyncio.get_event_loop().run_until_complete(yield_expression_coroutine())
+
+   def test_method_coroutine(self):
+
+   class Cubby(object):
+
+

[gentoo-commits] proj/portage:master commit in: pym/portage/repository/, man/, pym/portage/sync/modules/git/

2018-07-11 Thread Zac Medico

commit: 903c4b1a67689c4b8cc59113a56d58575cf7db8e
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Jul 10 07:03:35 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jul 11 06:10:41 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=903c4b1a

GitSync: support sync-depth (bug 552814)

Support sync-depth for shallow sync, using git reset --merge just
like in the earlier implementation that was reverted in commit
ab840ac982d3c8b676b89f6bedd14e85dd06870f. Also, run git gc --auto
in the foreground, in order to trigger periodic housekeeping and
hopefully avoid errors from automatic git gc calls as reported in
bug 599008.

The default sync-depth is unlimited, which means that default
behavior remains unchanged (unlike the previous implementation that
was reverted).

Bug: https://bugs.gentoo.org/552814
Bug: https://bugs.gentoo.org/599008

 man/portage.5   |  3 ++-
 pym/portage/repository/config.py|  4 
 pym/portage/sync/modules/git/git.py | 26 +-
 3 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/man/portage.5 b/man/portage.5
index acc80791b..a57531d44 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -985,7 +985,8 @@ overlay filesystems.
 Specifies CVS repository.
 .TP
 .B sync\-depth
-This is a deprecated alias for the \fBclone\-depth\fR option.
+Specifies sync depth to use for DVCS repositories. If set to 0, the
+depth is unlimited. Defaults to 0.
 .TP
 .B sync\-git\-clone\-env
 Set environment variables for git when cloning repository (git clone).

diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index ad7ae9d18..bf2b6dd03 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -179,10 +179,6 @@ class RepoConfig(object):
self.clone_depth = repo_opts.get('clone-depth')
self.sync_depth = repo_opts.get('sync-depth')
 
-   if self.sync_depth is not None:
-   warnings.warn(_("repos.conf: sync-depth is deprecated,"
-   " use clone-depth instead"))
-
self.sync_hooks_only_on_change = repo_opts.get(
'sync-hooks-only-on-change', 'false').lower() == 'true'
 

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index 68f8bd1fb..2fb82c600 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -137,6 +137,24 @@ class GitSync(NewBase):
writemsg_level(msg + "\n", level=logging.ERROR, 
noiselevel=-1)
return (e.returncode, False)
 
+   shallow = self.repo.sync_depth is not None and 
self.repo.sync_depth != 0
+   if shallow:
+   git_cmd_opts += " --depth %d" % self.repo.sync_depth
+
+   # For shallow fetch, unreachable objects may need to be 
pruned
+   # manually, in order to prevent automatic git gc calls 
from
+   # eventually failing (see bug 599008).
+   gc_cmd = ['git', '-c', 'gc.autodetach=false', 'gc', 
'--auto']
+   if quiet:
+   gc_cmd.append('--quiet')
+   exitcode = subprocess.call(gc_cmd,
+   cwd=portage._unicode_encode(self.repo.location))
+   if exitcode != os.EX_OK:
+   msg = "!!! git gc error in %s" % 
self.repo.location
+   self.logger(self.xterm_titles, msg)
+   writemsg_level(msg + "\n", level=logging.ERROR, 
noiselevel=-1)
+   return (exitcode, False)
+
git_cmd = "%s fetch %s%s" % (self.bin_command,
remote_branch.partition('/')[0], git_cmd_opts)
 
@@ -159,7 +177,13 @@ class GitSync(NewBase):
if not self.verify_head(revision='refs/remotes/%s' % 
remote_branch):
return (1, False)
 
-   merge_cmd = [self.bin_command, 'merge', 'refs/remotes/%s' % 
remote_branch]
+   if shallow:
+   # Since the default merge strategy typically fails when
+   # the depth is not unlimited, `git reset --merge`.
+   merge_cmd = [self.bin_command, 'reset', '--merge']
+   else:
+   merge_cmd = [self.bin_command, 'merge']
+   merge_cmd.append('refs/remotes/%s' % remote_branch)
if quiet:
merge_cmd.append('--quiet')
exitcode = subprocess.call(merge_cmd,

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/, cnf/, pym/portage/repository/, man/

2018-07-10 Thread Zac Medico

commit: 84822ef7a21494d3f044c2ffa7b112e4d29665ab
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Jul  5 13:10:43 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Jul 10 05:03:53 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=84822ef7

rsync: quarantine data prior to verification (bug 660410)

Sync into a quarantine subdirectory, using the rsync --link-dest option
to create hardlinks to identical files in the previous snapshot of the
repository. If hardlinks are not supported, then show a warning message
and sync directly to the normal repository location.

If verification succeeds, then the quarantine subdirectory is synced
to the normal repository location, and the quarantine subdirectory
is deleted. If verification fails, then the quarantine directory is
preserved for purposes of analysis.

Even if verification happens to be disabled, the quarantine directory
is still useful for making the repository update more atomic, so that
it is less likely that normal repository location will be observed in
a partially synced state.

The new behavior may conflict with configurations that restrict the
use of hardlinks, such as overlay filesystems. Therefore, users will
have to set "sync-allow-hardlinks = no" in repos.conf if they have
a configuration that prevents the use of hardlinks, but this should
not be very common.

Bug: https://bugs.gentoo.org/660410

 cnf/repos.conf  |  1 +
 man/portage.5   |  8 +++
 pym/portage/repository/config.py|  7 ++-
 pym/portage/sync/modules/rsync/rsync.py | 87 ++---
 4 files changed, 94 insertions(+), 9 deletions(-)

diff --git a/cnf/repos.conf b/cnf/repos.conf
index 352073cfd..419f6d118 100644
--- a/cnf/repos.conf
+++ b/cnf/repos.conf
@@ -6,6 +6,7 @@ location = /usr/portage
 sync-type = rsync
 sync-uri = rsync://rsync.gentoo.org/gentoo-portage
 auto-sync = yes
+sync-allow-hardlinks = yes
 sync-rsync-verify-jobs = 1
 sync-rsync-verify-metamanifest = yes
 sync-rsync-verify-max-age = 24

diff --git a/man/portage.5 b/man/portage.5
index 5adb07d82..acc80791b 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -973,6 +973,14 @@ files). Defaults to true.
 .br
 Valid values: true, false.
 .TP
+.B sync\-allow\-hardlinks = yes|no
+Allow sync plugins to use hardlinks in order to ensure that a repository
+remains in a valid state if something goes wrong during the sync operation.
+For example, if signature verification fails during a sync operation,
+the previous state of the repository will be preserved. This option may
+conflict with configurations that restrict the use of hardlinks, such as
+overlay filesystems.
+.TP
 .B sync\-cvs\-repo
 Specifies CVS repository.
 .TP

diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 1d897bb90..ad7ae9d18 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -86,6 +86,7 @@ class RepoConfig(object):
'sync_type', 'sync_umask', 'sync_uri', 'sync_user', 
'thin_manifest',
'update_changelog', '_eapis_banned', '_eapis_deprecated',
'_masters_orig', 'module_specific_options', 
'manifest_required_hashes',
+   'sync_allow_hardlinks',
'sync_openpgp_key_path',
'sync_openpgp_key_refresh_retry_count',
'sync_openpgp_key_refresh_retry_delay_max',
@@ -188,6 +189,9 @@ class RepoConfig(object):
self.strict_misc_digests = repo_opts.get(
'strict-misc-digests', 'true').lower() == 'true'
 
+   self.sync_allow_hardlinks = repo_opts.get(
+   'sync-allow-hardlinks', 'true').lower() in ('true', 
'yes')
+
self.sync_openpgp_key_path = repo_opts.get(
'sync-openpgp-key-path', None)
 
@@ -534,6 +538,7 @@ class RepoConfigLoader(object):
'clone_depth', 
'eclass_overrides',
'force', 'masters', 
'priority', 'strict_misc_digests',
'sync_depth', 
'sync_hooks_only_on_change',
+   'sync_allow_hardlinks',
'sync_openpgp_key_path',

'sync_openpgp_key_refresh_retry_count',

'sync_openpgp_key_refresh_retry_delay_max',
@@ -960,7 +965,7 @@ class RepoConfigLoader(object):
return repo_name in self.prepos
 
def config_string(self):
-   bool_keys = ("strict_misc_digests",)
+   bool_keys = ("strict_misc_digests", "sync_allow_hardlinks")
str_or_int_keys = ("auto_sync", "clone_depth", "format", 
"location",
"main_repo",

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/, pym/portage/sync/modules/rsync/

2018-07-09 Thread Zac Medico

commit: 06f304c8718ac653bbdf9b5b999c03cba898bf3e
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jul  8 20:09:45 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Jul 10 04:23:12 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=06f304c8

SyncBase: split out _refresh_keys method (bug 660732)

Split out a _refresh_keys method from the RsyncSync class,
so GitSync can use it for retry support.

Bug: https://bugs.gentoo.org/660732

 pym/portage/sync/modules/rsync/rsync.py | 33 +-
 pym/portage/sync/syncbase.py| 41 +
 2 files changed, 42 insertions(+), 32 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 382a1eaae..a715e2818 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -7,7 +7,6 @@ import time
 import signal
 import socket
 import datetime
-import functools
 import io
 import re
 import random
@@ -23,10 +22,8 @@ good = create_color_func("GOOD")
 bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
-from portage.util._eventloop.global_event_loop import global_event_loop
 from portage.util import writemsg, writemsg_stdout
 from portage.util.futures import asyncio
-from portage.util.futures.executor.fork import ForkExecutor
 from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import NewBase
@@ -149,39 +146,11 @@ class RsyncSync(NewBase):
# will not be performed and the user will have to fix 
it and try again,
# so we may as well bail out before actual rsync 
happens.
if openpgp_env is not None and 
self.repo.sync_openpgp_key_path is not None:
-
try:
out.einfo('Using keys from %s' % 
(self.repo.sync_openpgp_key_path,))
with 
io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
openpgp_env.import_key(f)
-   out.ebegin('Refreshing keys from 
keyserver')
-   retry_decorator = 
self._key_refresh_retry_decorator()
-   if retry_decorator is None:
-   openpgp_env.refresh_keys()
-   else:
-   def noisy_refresh_keys():
-   """
-   Since retry does not 
help for some types of
-   errors, display errors 
as soon as they occur.
-   """
-   try:
-   
openpgp_env.refresh_keys()
-   except Exception as e:
-   
writemsg_level("%s\n" % (e,),
-   
level=logging.ERROR, noiselevel=-1)
-   raise # retry
-
-   # The ThreadPoolExecutor that 
asyncio uses by default
-   # does not support cancellation 
of tasks, therefore
-   # use ForkExecutor for task 
cancellation support, in
-   # order to enforce timeouts.
-   loop = global_event_loop()
-   with ForkExecutor(loop=loop) as 
executor:
-   func_coroutine = 
functools.partial(loop.run_in_executor,
-   executor, 
noisy_refresh_keys)
-   decorated_func = 
retry_decorator(func_coroutine, loop=loop)
-   
loop.run_until_complete(decorated_func())
-   out.eend(0)
+   self._refresh_keys(openpgp_env)
except (GematoException, asyncio.TimeoutError) 
as e:
writemsg_level("!!! Manifest 
verification impossible due to keyring problem:\n%s\n"
% (e,),

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/

2018-07-09 Thread Zac Medico

commit: 89b85e47d7ac0d5f36b182c36eb1e72db7187b36
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jul  8 20:29:40 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Jul 10 04:25:54 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=89b85e47

GitSync: add key refresh retry (bug 660732)

Bug: https://bugs.gentoo.org/660732

 pym/portage/sync/modules/git/git.py | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index 97c4c1de6..68f8bd1fb 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -8,6 +8,7 @@ import subprocess
 import portage
 from portage import os
 from portage.util import writemsg_level, shlex_split
+from portage.util.futures import asyncio
 from portage.output import create_color_func, EOutput
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")
@@ -197,10 +198,8 @@ class GitSync(NewBase):
out.einfo('Using keys from %s' % 
(self.repo.sync_openpgp_key_path,))
with 
io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
openpgp_env.import_key(f)
-   out.ebegin('Refreshing keys from 
keyserver')
-   openpgp_env.refresh_keys()
-   out.eend(0)
-   except GematoException as e:
+   self._refresh_keys(openpgp_env)
+   except (GematoException, asyncio.TimeoutError) 
as e:
writemsg_level("!!! Verification 
impossible due to keyring problem:\n%s\n"
% (e,),
level=logging.ERROR, 
noiselevel=-1)

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/

2018-07-08 Thread Zac Medico

commit: dac12abb20592b098163c87572f6e04416ef3996
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jul  8 22:15:09 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jul  8 22:15:09 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dac12abb

GitSync: fix verification of remote branch

Fixes: 3cd8cf93abb6 ("GitSync: abort checkout for signature problem (bug 
660372)")

 pym/portage/sync/modules/git/git.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index 85a44289a..97c4c1de6 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -155,7 +155,7 @@ class GitSync(NewBase):
writemsg_level(msg + "\n", level=logging.ERROR, 
noiselevel=-1)
return (exitcode, False)
 
-   if not self.verify_head(revision='refs/remotes/%s^..' % 
remote_branch):
+   if not self.verify_head(revision='refs/remotes/%s' % 
remote_branch):
return (1, False)
 
merge_cmd = [self.bin_command, 'merge', 'refs/remotes/%s' % 
remote_branch]
@@ -209,7 +209,7 @@ class GitSync(NewBase):
env = os.environ.copy()
env['GNUPGHOME'] = openpgp_env.home
 
-   rev_cmd = [self.bin_command, "log", 
"--pretty=format:%G?", revision]
+   rev_cmd = [self.bin_command, "log", "-n1", 
"--pretty=format:%G?", revision]
try:
status = (portage._unicode_decode(
subprocess.check_output(rev_cmd,

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/

2018-07-08 Thread Zac Medico

commit: 3cd8cf93abb6410cc877381531bb662a704dffa7
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Jul  5 10:10:36 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jul  8 21:16:31 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3cd8cf93

GitSync: abort checkout for signature problem (bug 660372)

Fetch the upstream remote and use git merge to update the checkout
only after successful verification of the upstream head.

Suggested-by: Richard Freeman  gentoo.org>
Reviewed-by: Arfrever Frehtes Taifersar Arahesis  Apache.Org>
Bug: https://bugs.gentoo.org/660372

 pym/portage/sync/modules/git/git.py | 39 -
 1 file changed, 34 insertions(+), 5 deletions(-)

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index 160137a6d..85a44289a 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -109,6 +109,7 @@ class GitSync(NewBase):
if not self.has_bin:
return (1, False)
git_cmd_opts = ""
+   quiet = self.settings.get("PORTAGE_QUIET") == "1"
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = 
shlex_split(self.repo.module_specific_options['sync-git-env'])
env = dict((k, v) for k, _, v in 
(assignment.partition('=') for assignment in shlexed_env) if k)
@@ -123,7 +124,21 @@ class GitSync(NewBase):
git_cmd_opts += " --quiet"
if 
self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
git_cmd_opts += " %s" % 
self.repo.module_specific_options['sync-git-pull-extra-opts']
-   git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
+
+   try:
+   remote_branch = portage._unicode_decode(
+   subprocess.check_output([self.bin_command, 
'rev-parse',
+   '--abbrev-ref', '--symbolic-full-name', 
'@{upstream}'],
+   
cwd=portage._unicode_encode(self.repo.location))).rstrip('\n')
+   except subprocess.CalledProcessError as e:
+   msg = "!!! git rev-parse error in %s" % 
self.repo.location
+   self.logger(self.xterm_titles, msg)
+   writemsg_level(msg + "\n", level=logging.ERROR, 
noiselevel=-1)
+   return (e.returncode, False)
+
+   git_cmd = "%s fetch %s%s" % (self.bin_command,
+   remote_branch.partition('/')[0], git_cmd_opts)
+
writemsg_level(git_cmd + "\n")
 
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", 
"HEAD"]
@@ -133,20 +148,34 @@ class GitSync(NewBase):
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), 
git_cmd),
**self.spawn_kwargs)
+
if exitcode != os.EX_OK:
-   msg = "!!! git pull error in %s" % self.repo.location
+   msg = "!!! git fetch error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, 
noiselevel=-1)
return (exitcode, False)
-   if not self.verify_head():
+
+   if not self.verify_head(revision='refs/remotes/%s^..' % 
remote_branch):
return (1, False)
 
+   merge_cmd = [self.bin_command, 'merge', 'refs/remotes/%s' % 
remote_branch]
+   if quiet:
+   merge_cmd.append('--quiet')
+   exitcode = subprocess.call(merge_cmd,
+   cwd=portage._unicode_encode(self.repo.location))
+
+   if exitcode != os.EX_OK:
+   msg = "!!! git merge error in %s" % self.repo.location
+   self.logger(self.xterm_titles, msg)
+   writemsg_level(msg + "\n", level=logging.ERROR, 
noiselevel=-1)
+   return (exitcode, False)
+
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
 
return (os.EX_OK, current_rev != previous_rev)
 
-   def verify_head(self):
+   def verify_head(self, revision='-1'):
if (self.repo.module_specific_options.get(
'sync-git-verify-commit-signature', 'false') != 
'true'):
return True
@@ -180,7 +209,7 @@ class GitSync(NewBase):
env = os.environ.copy()
env['GNUPGHOME'] = openpgp_env.home
 
-   rev_cmd = [self.bin_command, "log", 
"--pretty=format:%G?", "-1"]
+

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/sync/

2018-07-05 Thread Zac Medico

commit: 82823b0c4de0a7cbb5654bb19d63aef874800afd
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Jul  5 10:18:55 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Jul  5 10:19:23 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=82823b0c

test_sync_local: fix GitSync coverage

Fixes: 0655b4a26e37 ("test_sync_local: add test for auto-sync set to 'no'")

 pym/portage/tests/sync/test_sync_local.py | 15 +--
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/pym/portage/tests/sync/test_sync_local.py 
b/pym/portage/tests/sync/test_sync_local.py
index 010c8f887..17ff6f200 100644
--- a/pym/portage/tests/sync/test_sync_local.py
+++ b/pym/portage/tests/sync/test_sync_local.py
@@ -102,17 +102,20 @@ class SyncLocalTestCase(TestCase):
os.unlink(os.path.join(metadata_dir, 'timestamp.chk'))
 
sync_cmds = (
+   (homedir, cmds["emerge"] + ("--sync",)),
+   (homedir, lambda: self.assertTrue(os.path.exists(
+   os.path.join(repo.location, "dev-libs", "A")
+   ), "dev-libs/A expected, but missing")),
+   (homedir, cmds["emaint"] + ("sync", "-A")),
+   )
+
+   sync_cmds_auto_sync = (
(homedir, lambda: repos_set_conf("rsync", 
auto_sync="no")),
(homedir, cmds["emerge"] + ("--sync",)),
(homedir, lambda: self.assertFalse(os.path.exists(
os.path.join(repo.location, "dev-libs", "A")
), "dev-libs/A found, expected missing")),
(homedir, lambda: repos_set_conf("rsync", 
auto_sync="yes")),
-   (homedir, cmds["emerge"] + ("--sync",)),
-   (homedir, lambda: self.assertTrue(os.path.exists(
-   os.path.join(repo.location, "dev-libs", "A")
-   ), "dev-libs/A expected, but missing")),
-   (homedir, cmds["emaint"] + ("sync", "-A")),
)
 
rename_repo = (
@@ -236,7 +239,7 @@ class SyncLocalTestCase(TestCase):
# triggered by python -Wd will be visible.
stdout = subprocess.PIPE
 
-   for cwd, cmd in rename_repo + sync_cmds + \
+   for cwd, cmd in rename_repo + sync_cmds_auto_sync + 
sync_cmds + \
rsync_opts_repos + rsync_opts_repos_default + \
rsync_opts_repos_default_ovr + 
rsync_opts_repos_default_cancel + \
delete_sync_repo + git_repo_create + 
sync_type_git + \

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/, pym/portage/util/

2018-06-26 Thread Zac Medico

commit: deb87a465306d05146d7eb55d27d7d89943725c0
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jun 24 21:42:52 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jun 27 03:15:08 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=deb87a46

{,PKG_}INSTALL_MASK: support trailing slash (bug 658322)

Fix the python INSTALL_MASK implementation so that a trailing slash
matches a directory, for compatibility with the previous bash
implementation.

Fixes: 3416876c0ee7 ("{,PKG_}INSTALL_MASK: python implementation")
Bug: https://bugs.gentoo.org/658322

 pym/portage/tests/util/test_install_mask.py | 129 
 pym/portage/util/install_mask.py|   7 +-
 2 files changed, 134 insertions(+), 2 deletions(-)

diff --git a/pym/portage/tests/util/test_install_mask.py 
b/pym/portage/tests/util/test_install_mask.py
new file mode 100644
index 0..f651eb4b7
--- /dev/null
+++ b/pym/portage/tests/util/test_install_mask.py
@@ -0,0 +1,129 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.util.install_mask import InstallMask
+
+
+class InstallMaskTestCase(TestCase):
+
+   def testTrailingSlash(self):
+   """
+   Test that elements with a trailing slash match a directory
+   but not a regular file.
+   """
+   cases = (
+   (
+   '/foo/bar/ -/foo/bar/*.foo -*.baz',
+   (
+   (
+   'foo/bar/baz',
+   True,
+   ),
+   (
+   'foo/bar/',
+   True,
+   ),
+   # /foo/bar/ does not match
+   (
+   'foo/bar',
+   False,
+   ),
+   # this is excluded
+   (
+   'foo/bar/baz.foo',
+   False,
+   ),
+   # this is excluded
+   (
+   'foo/bar/baz.baz',
+   False,
+   ),
+   (
+   'foo/bar/baz.bar',
+   True,
+   ),
+   )
+   ),
+   (
+   '/foo/bar -/foo/bar/*.foo -*.baz',
+   (
+   (
+   'foo/bar/baz',
+   True,
+   ),
+   # /foo/bar matches both foo/bar/ and 
foo/bar
+   (
+   'foo/bar/',
+   True,
+   ),
+   (
+   'foo/bar',
+   True,
+   ),
+   # this is excluded
+   (
+   'foo/bar/baz.foo',
+   False,
+   ),
+   # this is excluded
+   (
+   'foo/bar/baz.baz',
+   False,
+   ),
+   (
+   'foo/bar/baz.bar',
+   True,
+   ),
+   )
+   ),
+   (
+   '/foo*',
+   (
+   (
+   'foo',
+   True,
+   ),
+

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/

2018-06-26 Thread Zac Medico

commit: 6b50ba69f5a8e311fcddfb2e5c203631bd292c71
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Jun 21 21:03:38 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jun 27 03:04:38 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6b50ba69

AsyncioEventLoop: exit after unhandled exception (bug 658684)

Fix portage commands to exit immediately for any unhandled
exceptions that are raised while the asyncio event loop is running
without a tty. If we have a tty then start the debugger,
since in might aid in diagnosis of the problem.

In order to avoid potential interference with API consumers, do not
call set_exception_handler unless portage._internal_caller is True.

Bug: https://bugs.gentoo.org/658684

 pym/portage/util/_eventloop/asyncio_event_loop.py | 31 +++
 1 file changed, 31 insertions(+)

diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py 
b/pym/portage/util/_eventloop/asyncio_event_loop.py
index c07b71103..ea0e03b23 100644
--- a/pym/portage/util/_eventloop/asyncio_event_loop.py
+++ b/pym/portage/util/_eventloop/asyncio_event_loop.py
@@ -2,7 +2,9 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import os
+import pdb
 import signal
+import sys
 
 try:
import asyncio as _real_asyncio
@@ -53,6 +55,35 @@ class AsyncioEventLoop(_AbstractEventLoop):
self.get_debug = loop.get_debug
self._wakeup_fd = -1
 
+   if portage._internal_caller:
+   
loop.set_exception_handler(self._internal_caller_exception_handler)
+
+   @staticmethod
+   def _internal_caller_exception_handler(loop, context):
+   """
+   An exception handler which drops to a pdb shell if std* streams
+   refer to a tty, and otherwise kills the process with SIGTERM.
+
+   In order to avoid potential interference with API consumers, 
this
+   implementation is only used when portage._internal_caller is 
True.
+   """
+   loop.default_exception_handler(context)
+   if 'exception' in context:
+   # If we have a tty then start the debugger, since in 
might
+   # aid in diagnosis of the problem. If there's no tty, 
then
+   # exit immediately.
+   if all(s.isatty() for s in (sys.stdout, sys.stderr, 
sys.stdin)):
+   pdb.set_trace()
+   else:
+   # Normally emerge will wait for all coroutines 
to complete
+   # after SIGTERM has been received. However, an 
unhandled
+   # exception will prevent the interrupted 
coroutine from
+   # completing, therefore use the default SIGTERM 
handler
+   # in order to ensure that emerge exits 
immediately (though
+   # uncleanly).
+   signal.signal(signal.SIGTERM, signal.SIG_DFL)
+   os.kill(os.getpid(), signal.SIGTERM)
+
def _create_future(self):
"""
Provide AbstractEventLoop.create_future() for python3.4.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

2018-06-26 Thread Zac Medico

commit: 2710f9ab525e7c726c2ffb027e242dbdf31cfe76
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Jun 26 20:14:09 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Jun 26 20:16:25 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2710f9ab

binarytree._merge_pkgindex_header: deduplicate values (bug 657422)

Deduplicate values of implicit IUSE variables, in order to
prevent them from growing without bound, triggering "[Errno 7]
Argument list too long" errors as reported in bug 657422.

Fixes: cab78dba98c4 ("emerge --usepkgonly: propagate implicit IUSE and 
USE_EXPAND (bug 640318)")
Bug: https://bugs.gentoo.org/657422

 pym/portage/dbapi/bintree.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 269a7b226..9c2d877e7 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1320,7 +1320,9 @@ class binarytree(object):
for k, v in iter_iuse_vars(src):
v_before = dest.get(k)
if v_before is not None:
-   v = v_before + ' ' + v
+   merged_values = set(v_before.split())
+   merged_values.update(v.split())
+   v = ' '.join(sorted(merged_values))
dest[k] = v
 
if 'ARCH' not in dest and 'ARCH' in src:

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/

2018-06-23 Thread Ulrich Müller

commit: 294f1a18f3271f7d506d346c2a514a2afa6ce8ec
Author: Ulrich Müller  gentoo  org>
AuthorDate: Sat Jun 23 09:22:18 2018 +
Commit: Ulrich Müller  gentoo  org>
CommitDate: Sat Jun 23 09:22:18 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=294f1a18

Revert "_post_src_install_uid_fix: allow files with portage group permissions 
(bug 600804)"

This reverts commit f479a4cdcac5db92231f489f232f10eb934c6f12.
Acked by zmedico in #gentoo-portage.

Bug: https://bugs.gentoo.org/600804

 pym/portage/package/ebuild/doebuild.py | 9 +++--
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/pym/portage/package/ebuild/doebuild.py 
b/pym/portage/package/ebuild/doebuild.py
index 0e94de805..97a9199a3 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -2045,7 +2045,7 @@ def _postinst_bsdflags(mysettings):
 def _post_src_install_uid_fix(mysettings, out):
"""
Files in $D with user and group bits that match the "portage"
-   user and group are automatically mapped to PORTAGE_INST_UID and
+   user or group are automatically mapped to PORTAGE_INST_UID and
PORTAGE_INST_GID if necessary. The chown system call may clear
S_ISUID and S_ISGID bits, so those bits are restored if
necessary.
@@ -2191,11 +2191,8 @@ def _post_src_install_uid_fix(mysettings, out):
mystat.st_ino not in counted_inodes:
counted_inodes.add(mystat.st_ino)
size += mystat.st_size
-
-   # Only remap the UID/GID if both match the 
portage user,
-   # in order to avoid interference with ebuilds 
that install
-   # files with portage group permissions (see bug 
600804).
-   if (mystat.st_uid, mystat.st_gid) != 
(portage_uid, portage_gid):
+   if mystat.st_uid != portage_uid and \
+   mystat.st_gid != portage_gid:
continue
myuid = -1
mygid = -1

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/, pym/_emerge/

2018-06-04 Thread Zac Medico

commit: 345256c2d439c5ab580e4226f227db2819883d40
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Jun  4 23:32:46 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Jun  4 23:44:15 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=345256c2

emerge -pf: spawn pkg_nofetch asynchronously (bug 657360)

For pretend mode, fix doebuild to skip the spawn_nofetch call
that would trigger event loop recursion, and spawn pkg_nofetch
asynchronously.

Bug: https://bugs.gentoo.org/657360

 pym/_emerge/EbuildBuild.py | 16 ++--
 pym/_emerge/EbuildFetchonly.py |  5 -
 pym/portage/package/ebuild/doebuild.py | 15 +++
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py
index 00d4680f5..d9f7f6da7 100644
--- a/pym/_emerge/EbuildBuild.py
+++ b/pym/_emerge/EbuildBuild.py
@@ -142,8 +142,20 @@ class EbuildBuild(CompositeTask):
pkg=pkg, pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
-   self.returncode = retval
-   self.wait()
+   if retval == os.EX_OK:
+   self._current_task = None
+   self.returncode = os.EX_OK
+   self._async_wait()
+   else:
+   # For pretend mode, the convention it 
to execute
+   # pkg_nofetch and return a successful 
exitcode.
+   
self._start_task(SpawnNofetchWithoutBuilddir(
+   background=self.background,
+   
portdb=self.pkg.root_config.trees[self._tree].dbapi,
+   ebuild_path=self._ebuild_path,
+   scheduler=self.scheduler,
+   settings=self.settings),
+   self._default_final_exit)
return
else:
fetcher = EbuildFetcher(

diff --git a/pym/_emerge/EbuildFetchonly.py b/pym/_emerge/EbuildFetchonly.py
index f88ea96ef..eec2ad208 100644
--- a/pym/_emerge/EbuildFetchonly.py
+++ b/pym/_emerge/EbuildFetchonly.py
@@ -25,7 +25,10 @@ class EbuildFetchonly(SlotObject):
listonly=self.pretend, fetchonly=1, 
fetchall=self.fetch_all,
mydbapi=portdb, tree="porttree")
 
-   if rval != os.EX_OK:
+   # For pretend mode, this error message is suppressed,
+   # and the unsuccessful return value is used to trigger
+   # a call to the pkg_nofetch phase.
+   if rval != os.EX_OK and not self.pretend:
msg = "Fetch failed for '%s'" % (pkg.cpv,)
eerror(msg, phase="unpack", key=pkg.cpv)
 

diff --git a/pym/portage/package/ebuild/doebuild.py 
b/pym/portage/package/ebuild/doebuild.py
index dc443df00..0e94de805 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -1079,14 +1079,13 @@ def doebuild(myebuild, mydo, 
_unused=DeprecationWarning, settings=None, debug=0,
if not fetch(fetchme, mysettings, listonly=listonly,
fetchonly=fetchonly, 
allow_missing_digests=False,
digests=dist_digests):
-   spawn_nofetch(mydbapi, myebuild, 
settings=mysettings,
-   fd_pipes=fd_pipes)
-   if listonly:
-   # The convention for listonly mode is 
to report
-   # success in any case, even though 
fetch() may
-   # return unsuccessfully in order to 
trigger the
-   # nofetch phase.
-   return 0
+   # Since listonly mode is called by emerge 
--pretend in an
+   # asynchronous context, spawn_nofetch would 
trigger event loop
+   # recursion here, therefore delegate execution 
of pkg_nofetch
+   # to the caller (bug 657360).
+   if not listonly:
+   spawn_nofetch(mydbapi, myebuild, 
settings=mysettings,
+   fd_pipes=fd_pipes)
return 1
 
if need_distfiles:

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/process/

2018-05-26 Thread Zac Medico

commit: ac5b48b253add3007034f9fdad02779cd3972281
Author: Zac Medico  gentoo  org>
AuthorDate: Sat May 26 22:38:58 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat May 26 22:38:58 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ac5b48b2

PipeReaderTestCase: wait for consumer

 pym/portage/tests/process/test_poll.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pym/portage/tests/process/test_poll.py 
b/pym/portage/tests/process/test_poll.py
index d71c9b59c..f700a5585 100644
--- a/pym/portage/tests/process/test_poll.py
+++ b/pym/portage/tests/process/test_poll.py
@@ -78,6 +78,7 @@ class PipeReaderTestCase(TestCase):
producer.start()
os.close(slave_fd)
producer.wait()
+   consumer.wait()
 
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/

2018-05-26 Thread Zac Medico

commit: 4fb5ef2ce2cb27ae155a25bfa5a4666597afb6ac
Author: Zac Medico  gentoo  org>
AuthorDate: Sat May 26 20:20:06 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat May 26 20:20:06 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4fb5ef2c

EventLoop.close: fix 'AttributeError: close' for python2

For python2 without epoll, fix handling of missing
'close' attribute on self._poll_obj.

Fixes: 4095be74985c ("Add ForkExecutor (bug 649588)")
Reported-by: Brian Evans  gentoo.org>

 pym/portage/util/_eventloop/EventLoop.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/util/_eventloop/EventLoop.py 
b/pym/portage/util/_eventloop/EventLoop.py
index de0795224..df76374d9 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -968,7 +968,7 @@ class EventLoop(object):
executor.shutdown(wait=True)
 
if self._poll_obj is not None:
-   close = getattr(self._poll_obj, 'close')
+   close = getattr(self._poll_obj, 'close', None)
if close is not None:
close()
self._poll_obj = None

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/process/

2018-05-26 Thread Zac Medico

commit: 3b0c52b9ef364dc8e69208ec5341255ac94d41d8
Author: Zac Medico  gentoo  org>
AuthorDate: Sat May 26 10:44:38 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat May 26 10:44:38 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3b0c52b9

PipeReaderTestCase: cover sockets and named pipes

 pym/portage/tests/process/test_poll.py | 74 ++
 1 file changed, 49 insertions(+), 25 deletions(-)

diff --git a/pym/portage/tests/process/test_poll.py 
b/pym/portage/tests/process/test_poll.py
index 596ea3088..d71c9b59c 100644
--- a/pym/portage/tests/process/test_poll.py
+++ b/pym/portage/tests/process/test_poll.py
@@ -1,11 +1,16 @@
-# Copyright 1998-2013 Gentoo Foundation
+# Copyright 1998-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import functools
+import pty
+import shutil
+import socket
+import sys
 import subprocess
+import tempfile
 
 from portage import os
 from portage.tests import TestCase
-from portage.util._pty import _create_pty_or_pipe
 from portage.util._async.PopenProcess import PopenProcess
 from portage.util._eventloop.global_event_loop import global_event_loop
 from _emerge.PipeReader import PipeReader
@@ -13,28 +18,47 @@ from _emerge.PipeReader import PipeReader
 class PipeReaderTestCase(TestCase):
 
_use_array = False
-   _use_pty = False
_echo_cmd = "echo -n '%s'"
 
-   def _testPipeReader(self, test_string):
+   def test_pipe(self):
+   def make_pipes():
+   return os.pipe(), None
+   self._do_test(make_pipes)
+
+   def test_pty_device(self):
+   def make_pipes():
+   try:
+   return pty.openpty(), None
+   except EnvironmentError:
+   self.skipTest('pty not available')
+   self._do_test(make_pipes)
+
+   def test_domain_socket(self):
+   def make_pipes():
+   if sys.version_info >= (3, 2):
+   read_end, write_end = socket.socketpair()
+   return (read_end.detach(), write_end.detach()), 
None
+   else:
+   self.skipTest('socket detach not supported')
+   self._do_test(make_pipes)
+
+   def test_named_pipe(self):
+   def make_pipes():
+   tempdir = tempfile.mkdtemp()
+   fifo_path = os.path.join(tempdir, 'fifo')
+   os.mkfifo(fifo_path)
+   return ((os.open(fifo_path, os.O_NONBLOCK|os.O_RDONLY),
+   os.open(fifo_path, os.O_NONBLOCK|os.O_WRONLY)),
+   functools.partial(shutil.rmtree, tempdir))
+   self._do_test(make_pipes)
+
+   def _testPipeReader(self, master_fd, slave_fd, test_string):
"""
Use a poll loop to read data from a pipe and assert that
the data written to the pipe is identical to the data
read from the pipe.
"""
 
-   if self._use_pty:
-   got_pty, master_fd, slave_fd = _create_pty_or_pipe()
-   if not got_pty:
-   os.close(slave_fd)
-   os.close(master_fd)
-   skip_reason = "pty not acquired"
-   self.portage_skip = skip_reason
-   self.fail(skip_reason)
-   return
-   else:
-   master_fd, slave_fd = os.pipe()
-
# WARNING: It is very important to use unbuffered mode here,
# in order to avoid issue 5380 with python3.
master_file = os.fdopen(master_fd, 'rb', 0)
@@ -60,15 +84,18 @@ class PipeReaderTestCase(TestCase):
 
return consumer.getvalue().decode('ascii', 'replace')
 
-   def testPipeReader(self):
+   def _do_test(self, make_pipes):
for x in (1, 2, 5, 6, 7, 8, 2**5, 2**10, 2**12, 2**13, 2**14):
test_string = x * "a"
-   output = self._testPipeReader(test_string)
-   self.assertEqual(test_string, output,
-   "x = %s, len(output) = %s" % (x, len(output)))
+   (read_end, write_end), cleanup = make_pipes()
+   try:
+   output = self._testPipeReader(read_end, 
write_end, test_string)
+   self.assertEqual(test_string, output,
+   "x = %s, len(output) = %s" % (x, 
len(output)))
+   finally:
+   if cleanup is not None:
+   cleanup()
 
-class

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/

2018-05-26 Thread Zac Medico

commit: cd44fea4c8c8154fee74bcc37529c0d5c2e9d789
Author: Zac Medico  gentoo  org>
AuthorDate: Sat May 26 05:46:36 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat May 26 06:07:55 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cd44fea4

prepare_build_dirs: convert PermissionError to PermissionDenied (bug 656542)

This supresses display of a big PermissionError traceback for
bin/ebuild, like in https://bugs.gentoo.org/656542#c0, and also
suppresses additional messages that may occur when the unhandled
PermissionError prevents bin/ebuild from cleanly closing the asyncio
event loop.

Bug: https://bugs.gentoo.org/656542

 pym/portage/package/ebuild/prepare_build_dirs.py | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/portage/package/ebuild/prepare_build_dirs.py 
b/pym/portage/package/ebuild/prepare_build_dirs.py
index 21e2aa87c..e53ccd0fb 100644
--- a/pym/portage/package/ebuild/prepare_build_dirs.py
+++ b/pym/portage/package/ebuild/prepare_build_dirs.py
@@ -19,6 +19,7 @@ from portage.localization import _
 from portage.output import colorize
 from portage.util import apply_recursive_permissions, \
apply_secpass_permissions, ensure_dirs, normalize_path, writemsg
+from portage.util.install_mask import _raise_exc
 from portage.const import EPREFIX
 
 def prepare_build_dirs(myroot=None, settings=None, cleanup=False):
@@ -50,7 +51,9 @@ def prepare_build_dirs(myroot=None, settings=None, 
cleanup=False):
clean_dir, noiselevel=-1)
return 1
else:
-   raise
+   # Wrap with PermissionDenied if appropriate, so 
that callers
+   # display a short error message without a 
traceback.
+   _raise_exc(oe)
 
def makedirs(dir_path):
try:

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/

2018-05-25 Thread Zac Medico

commit: 466cc6c60fcf1d56049af7a1d37fc959f6808351
Author: Zac Medico  gentoo  org>
AuthorDate: Fri May 25 16:07:33 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Fri May 25 16:10:34 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=466cc6c6

SonameDepsProcessor: fix 'set' object has no attribute 'items' (bug 656492)

Use collections.defaultdict to ensure that requires_map
contains correct defaults.

Fixes: 1364cd44e7a6 ("SonameDepsProcessor: handle internal libs without 
DT_SONAME (bug 646190)")
Bug: https://bugs.gentoo.org/656492

 pym/portage/util/_dyn_libs/soname_deps.py | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/pym/portage/util/_dyn_libs/soname_deps.py 
b/pym/portage/util/_dyn_libs/soname_deps.py
index c6302afc2..544cbc8f1 100644
--- a/pym/portage/util/_dyn_libs/soname_deps.py
+++ b/pym/portage/util/_dyn_libs/soname_deps.py
@@ -3,7 +3,9 @@
 
 from __future__ import unicode_literals
 
+import collections
 import fnmatch
+import functools
 from itertools import chain
 import os
 import re
@@ -33,7 +35,8 @@ class SonameDepsProcessor(object):
"""
self._provides_exclude = self._exclude_pattern(provides_exclude)
self._requires_exclude = self._exclude_pattern(requires_exclude)
-   self._requires_map = {}
+   self._requires_map = collections.defaultdict(
+   functools.partial(collections.defaultdict, set))
self._provides_map = {}
self._provides_unfiltered = {}
self._basename_map = {}
@@ -84,8 +87,7 @@ class SonameDepsProcessor(object):
for x in entry.needed:
if (self._requires_exclude is None or
self._requires_exclude.match(x) is 
None):
-   self._requires_map.setdefault(
-   multilib_cat, {}).setdefault(x, 
set()).add(runpaths)
+   
self._requires_map[multilib_cat][x].add(runpaths)
 
if entry.soname:
self._provides_unfiltered.setdefault(
@@ -105,7 +107,6 @@ class SonameDepsProcessor(object):
provides_unfiltered = self._provides_unfiltered
 
for multilib_cat in set(chain(requires_map, provides_map)):
-   requires_map.setdefault(multilib_cat, set())
provides_map.setdefault(multilib_cat, set())
provides_unfiltered.setdefault(multilib_cat, set())
for soname, consumers in 
list(requires_map[multilib_cat].items()):

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/

2018-05-24 Thread Zac Medico

commit: 8a8f527c7587bf388645ad703e0305797a26c3b4
Author: Zac Medico  gentoo  org>
AuthorDate: Fri May 25 03:06:08 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Fri May 25 03:06:32 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8a8f527c

AsyncioEventLoop: remove redundant set_wakeup_fd call

 pym/portage/util/_eventloop/asyncio_event_loop.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py 
b/pym/portage/util/_eventloop/asyncio_event_loop.py
index 65b354544..c07b71103 100644
--- a/pym/portage/util/_eventloop/asyncio_event_loop.py
+++ b/pym/portage/util/_eventloop/asyncio_event_loop.py
@@ -104,5 +104,3 @@ class AsyncioEventLoop(_AbstractEventLoop):
return self._loop.run_until_complete(future)
finally:
self._wakeup_fd = signal.set_wakeup_fd(-1)
-   if self._wakeup_fd != -1:
-   signal.set_wakeup_fd(-1)

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/, pym/portage/util/_eventloop/

2018-05-24 Thread Zac Medico

commit: adee194534f0b3d9762efd1e8e8713c316b93f5a
Author: Zac Medico  gentoo  org>
AuthorDate: Thu May 24 22:36:29 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Fri May 25 02:01:27 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=adee1945

AsyncioEventLoop: suppress BlockingIOError warning (bug 655656)

Override AbstractEventLoop.run_until_complete() to prevent
BlockingIOError from occurring when the event loop is not running,
by using signal.set_wakeup_fd(-1) to temporarily disable the wakeup
fd. In order to avoid potential interference with API consumers,
only modify wakeup fd when portage._interal_caller is True.

Bug: https://bugs.gentoo.org/655656

 .../util/futures/asyncio/test_wakeup_fd_sigchld.py | 76 ++
 pym/portage/util/_eventloop/asyncio_event_loop.py  | 37 +--
 2 files changed, 106 insertions(+), 7 deletions(-)

diff --git a/pym/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py 
b/pym/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
new file mode 100644
index 0..abc67c241
--- /dev/null
+++ b/pym/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
@@ -0,0 +1,76 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import subprocess
+
+import portage
+from portage.const import PORTAGE_PYM_PATH
+from portage.tests import TestCase
+from portage.util._eventloop.global_event_loop import _asyncio_enabled
+
+
+class WakeupFdSigchldTestCase(TestCase):
+   def testWakeupFdSigchld(self):
+   """
+   This is expected to trigger a bunch of messages like the 
following
+   unless the fix for bug 655656 works as intended:
+
+   Exception ignored when trying to write to the signal wakeup fd:
+   BlockingIOError: [Errno 11] Resource temporarily unavailable
+   """
+   if not _asyncio_enabled:
+   self.skipTest('asyncio not enabled')
+
+   script = """
+import asyncio as _real_asyncio
+import os
+import signal
+import sys
+
+import portage
+
+# In order to avoid potential interference with API consumers, wakeup
+# fd handling is enabled only when portage._interal_caller is True.
+portage._internal_caller = True
+
+from portage.util.futures import asyncio
+
+loop = asyncio._wrap_loop()
+
+# Cause the loop to register a child watcher.
+proc = loop.run_until_complete(_real_asyncio.create_subprocess_exec('sleep', 
'0'))
+loop.run_until_complete(proc.wait())
+
+for i in range(8192):
+   os.kill(os.getpid(), signal.SIGCHLD)
+
+# Verify that the child watcher still works correctly
+# (this will hang if it doesn't).
+proc = loop.run_until_complete(_real_asyncio.create_subprocess_exec('sleep', 
'0'))
+loop.run_until_complete(proc.wait())
+loop.close()
+sys.stdout.write('success')
+sys.exit(os.EX_OK)
+"""
+
+   pythonpath = os.environ.get('PYTHONPATH', '').strip().split(':')
+   if not pythonpath or pythonpath[0] != PORTAGE_PYM_PATH:
+   pythonpath = [PORTAGE_PYM_PATH] + pythonpath
+   pythonpath = ':'.join(filter(None, pythonpath))
+
+   proc = subprocess.Popen(
+   [portage._python_interpreter, '-c', script],
+   stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+   env=dict(os.environ, PYTHONPATH=pythonpath))
+
+   out, err = proc.communicate()
+   try:
+   self.assertEqual(out[:100], b'success')
+   except Exception:
+   portage.writemsg(''.join('{}\n'.format(line)
+   for line in 
out.decode(errors='replace').splitlines()[:50]),
+   noiselevel=-1)
+   raise
+
+   self.assertEqual(proc.wait(), os.EX_OK)

diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py 
b/pym/portage/util/_eventloop/asyncio_event_loop.py
index bf5937de8..65b354544 100644
--- a/pym/portage/util/_eventloop/asyncio_event_loop.py
+++ b/pym/portage/util/_eventloop/asyncio_event_loop.py
@@ -1,6 +1,7 @@
 # Copyright 2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import os
 import signal
 
 try:
@@ -11,6 +12,8 @@ except ImportError:
_real_asyncio = None
_AbstractEventLoop = object
 
+import portage
+
 
 class AsyncioEventLoop(_AbstractEventLoop):
"""
@@ -26,13 +29,15 @@ class AsyncioEventLoop(_AbstractEventLoop):
def __init__(self, loop=None):
loop = loop or _real_asyncio.get_event_loop()
self._loop = loop
-   self.run_until_complete = loop.run_until_complete
+   self.run_until_complete = (self._run_until_complete
+   if portage._internal_caller else 
loop.run_until_complete)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/, pym/portage/tests/util/dyn_libs/

2018-05-24 Thread Zac Medico

commit: 1364cd44e7a6232bf425c4573b5bd3d6738d49a4
Author: Zac Medico  gentoo  org>
AuthorDate: Sun May 20 06:40:19 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu May 24 20:31:02 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1364cd44

SonameDepsProcessor: handle internal libs without DT_SONAME (bug 646190)

Packages like ebtables have internal libraries that lack a DT_SONAME
field in their ELF header. Consumers of these internal libraries have
DT_RUNPATH entries that refer to the directory containing the internal
libraries. For library dependencies that are satisfied by internal
libraries like this, it is inappropriate for SonameDepsProcessor to
include these depenedencies in the REQUIRES metadata, therefore fix
SonameDepsProcessor to automatically detect this case and exclude
these dependencies from the REQUIRES metadata. This solves incorrect
reporting of broken soname dependencies like the following:

$ emerge -p --depclean --ignore-soname-deps=n

Calculating dependencies... done!
 * Broken soname dependencies found:
 *
 *   x86_64: libebt_redirect.so required by:
 * net-firewall/ebtables-2.0.10.4
 *
 *   x86_64: libebt_log.so required by:
 * net-firewall/ebtables-2.0.10.4

Bug: https://bugs.gentoo.org/646190

 pym/portage/tests/util/dyn_libs/__init__.py|  0
 pym/portage/tests/util/dyn_libs/__test__.py|  0
 .../tests/util/dyn_libs/test_soname_deps.py| 34 +
 pym/portage/util/_dyn_libs/soname_deps.py  | 35 --
 4 files changed, 66 insertions(+), 3 deletions(-)

diff --git a/pym/portage/tests/util/dyn_libs/__init__.py 
b/pym/portage/tests/util/dyn_libs/__init__.py
new file mode 100644
index 0..e69de29bb

diff --git a/pym/portage/tests/util/dyn_libs/__test__.py 
b/pym/portage/tests/util/dyn_libs/__test__.py
new file mode 100644
index 0..e69de29bb

diff --git a/pym/portage/tests/util/dyn_libs/test_soname_deps.py 
b/pym/portage/tests/util/dyn_libs/test_soname_deps.py
new file mode 100644
index 0..823890c91
--- /dev/null
+++ b/pym/portage/tests/util/dyn_libs/test_soname_deps.py
@@ -0,0 +1,34 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.util._dyn_libs.NeededEntry import NeededEntry
+from portage.util._dyn_libs.soname_deps import SonameDepsProcessor
+
+
+class SonameDepsProcessorTestCase(TestCase):
+
+   def testInternalLibsWithoutSoname(self):
+   """
+   Test handling of internal libraries that lack an soname, which 
are
+   resolved via DT_RUNPATH, see ebtables for example (bug 646190).
+   """
+   needed_elf_2 = """
+X86_64;/sbin/ebtables;;/lib64/ebtables;libebt_802_3.so,libebtable_broute.so,libc.so.6;x86_64
+X86_64;/lib64/ebtables/libebtable_broute.so;;;libc.so.6;x86_64
+X86_64;/lib64/ebtables/libebt_802_3.so;;;libc.so.6;x86_64
+"""
+   soname_deps = SonameDepsProcessor('', '')
+
+   for line in needed_elf_2.splitlines():
+   if not line:
+   continue
+   entry = NeededEntry.parse(None, line)
+   soname_deps.add(entry)
+
+   self.assertEqual(soname_deps.provides, None)
+   # Prior to the fix for bug 646190, REQUIRES contained 
references to
+   # the internal libebt* libraries which are resolved via a 
DT_RUNPATH
+   # entry referring to the /lib64/ebtables directory that 
contains the
+   # internal libraries.
+   self.assertEqual(soname_deps.requires, 'x86_64: libc.so.6\n')

diff --git a/pym/portage/util/_dyn_libs/soname_deps.py 
b/pym/portage/util/_dyn_libs/soname_deps.py
index a7d595429..c6302afc2 100644
--- a/pym/portage/util/_dyn_libs/soname_deps.py
+++ b/pym/portage/util/_dyn_libs/soname_deps.py
@@ -9,6 +9,11 @@ import os
 import re
 
 from portage.util import shlex_split
+from portage.util import (
+   normalize_path,
+   varexpand,
+)
+
 
 class SonameDepsProcessor(object):
"""
@@ -31,6 +36,7 @@ class SonameDepsProcessor(object):
self._requires_map = {}
self._provides_map = {}
self._provides_unfiltered = {}
+   self._basename_map = {}
self._provides = None
self._requires = None
self._intersected = False
@@ -62,15 +68,24 @@ class SonameDepsProcessor(object):
raise AssertionError(
"Missing multilib category data: %s" % 
entry.filename)
 
+   self._basename_map.setdefault(
+   os.path.basename(entry.filename), []).append(entry)
+
if entry.needed and (
self._requires_exclude is None or
self._requires_exclude.match(

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/

2018-05-24 Thread Zac Medico

commit: 28d3c9ed46b16757626b292dd6d7a6cae054b669
Author: Zac Medico  gentoo  org>
AuthorDate: Thu May 24 19:40:10 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu May 24 19:42:13 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=28d3c9ed

doebuild: eliminate redundant aux_get calls (bug 656394)

These redundant aux_get calls triggered event loop recursion
when called via EbuildFetchonly as reported in bug 656394.

Bug: https://bugs.gentoo.org/656394

 pym/portage/package/ebuild/doebuild.py | 19 +--
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/pym/portage/package/ebuild/doebuild.py 
b/pym/portage/package/ebuild/doebuild.py
index c3b89ade2..dc443df00 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -1048,9 +1048,11 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, 
settings=None, debug=0,
mydo not in ("digest", "manifest") and "noauto" not in 
features)
if need_distfiles:
 
-   src_uri, = mydbapi.aux_get(mysettings.mycpv,
-   ["SRC_URI"], 
mytree=os.path.dirname(os.path.dirname(
-   os.path.dirname(myebuild
+   src_uri = mysettings.configdict["pkg"].get("SRC_URI")
+   if src_uri is None:
+   src_uri, = mydbapi.aux_get(mysettings.mycpv,
+   ["SRC_URI"], 
mytree=os.path.dirname(os.path.dirname(
+   os.path.dirname(myebuild
metadata = {
"EAPI": mysettings["EAPI"],
"SRC_URI" : src_uri,
@@ -1410,9 +1412,14 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
all_keys = set(Package.metadata_keys)
all_keys.add("SRC_URI")
all_keys = tuple(all_keys)
-   metadata = dict(zip(all_keys,
-   mydbapi.aux_get(mysettings.mycpv, all_keys,
-   myrepo=mysettings.get("PORTAGE_REPO_NAME"
+   metadata = mysettings.configdict['pkg']
+   if all(k in metadata for k in ("PORTAGE_REPO_NAME", "SRC_URI")):
+   metadata = dict(((k, metadata[k]) for k in all_keys if k in 
metadata),
+   repository=metadata["PORTAGE_REPO_NAME"])
+   else:
+   metadata = dict(zip(all_keys,
+   mydbapi.aux_get(mysettings.mycpv, all_keys,
+   myrepo=mysettings.get("PORTAGE_REPO_NAME"
 
class FakeTree(object):
def __init__(self, mydb):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/portage/

2018-05-17 Thread Zac Medico

commit: b2a5f03abc0c172b6189226e6f9a7491a002cf51
Author: Zac Medico  gentoo  org>
AuthorDate: Thu May 17 18:38:27 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu May 17 18:38:27 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2a5f03a

MergeProcess,spawn: unregister SIGCHLD and wakeup_fd (bug 655656)

In order to prevent forked processes from invoking the parent process's
SIGCHLD handler and writing to wakeup_fd (triggering BlockingIOError),
unregister the SIGCHLD and wakeup_fd.

Bug: https://bugs.gentoo.org/655656
Reported-by: Helmut Jarausch  igpm.rwth-aachen.de>

 pym/portage/dbapi/_MergeProcess.py | 10 ++
 pym/portage/process.py | 10 ++
 2 files changed, 20 insertions(+)

diff --git a/pym/portage/dbapi/_MergeProcess.py 
b/pym/portage/dbapi/_MergeProcess.py
index fefdf8635..371550079 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -178,6 +178,16 @@ class MergeProcess(ForkProcess):
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
 
+   # Unregister SIGCHLD handler and wakeup_fd for the 
parent
+   # process's event loop (bug 655656).
+   signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+   try:
+   wakeup_fd = signal.set_wakeup_fd(-1)
+   if wakeup_fd > 0:
+   os.close(wakeup_fd)
+   except (ValueError, OSError):
+   pass
+
portage.locks._close_fds()
# We don't exec, so use close_fds=False
# (see _setup_pipes docstring).

diff --git a/pym/portage/process.py b/pym/portage/process.py
index 2af783e22..fd326731a 100644
--- a/pym/portage/process.py
+++ b/pym/portage/process.py
@@ -472,6 +472,16 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, 
groups, uid, umask,
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
 
+   # Unregister SIGCHLD handler and wakeup_fd for the parent
+   # process's event loop (bug 655656).
+   signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+   try:
+   wakeup_fd = signal.set_wakeup_fd(-1)
+   if wakeup_fd > 0:
+   os.close(wakeup_fd)
+   except (ValueError, OSError):
+   pass
+
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-05-14 Thread Zac Medico

commit: b28b01a78e4cc0199111e804af5b56a33319356d
Author: Zac Medico  gentoo  org>
AuthorDate: Mon May 14 15:33:15 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon May 14 15:35:53 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b28b01a7

ForkProcess: unregister SIGCHLD and wakeup_fd (bug 655656)

In order to prevent forked processes from invoking the parent process's
SIGCHLD handler and writing to wakeup_fd (triggering BlockingIOError),
unregister the SIGCHLD and wakeup_fd.

Bug: https://bugs.gentoo.org/655656
Reported-by: Helmut Jarausch  igpm.rwth-aachen.de>

 pym/portage/util/_async/ForkProcess.py | 10 ++
 1 file changed, 10 insertions(+)

diff --git a/pym/portage/util/_async/ForkProcess.py 
b/pym/portage/util/_async/ForkProcess.py
index 25f72d308..d84e93833 100644
--- a/pym/portage/util/_async/ForkProcess.py
+++ b/pym/portage/util/_async/ForkProcess.py
@@ -37,6 +37,16 @@ class ForkProcess(SpawnProcess):
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
 
+   # Unregister SIGCHLD handler and wakeup_fd for 
the parent
+   # process's event loop (bug 655656).
+   signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+   try:
+   wakeup_fd = signal.set_wakeup_fd(-1)
+   if wakeup_fd > 0:
+   os.close(wakeup_fd)
+   except (ValueError, OSError):
+   pass
+
portage.locks._close_fds()
# We don't exec, so use close_fds=False
# (see _setup_pipes docstring).

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/

2018-05-13 Thread Zac Medico

commit: 65379f436759dfbc4d56e52f1a145950779ebb60
Author: Zac Medico  gentoo  org>
AuthorDate: Sun May 13 16:45:27 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun May 13 16:57:38 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=65379f43

AsyncioEventLoop: suppress BlockingIOError warning during loop close (bug 
655656)

Disable the asyncio event loop's SIGCHLD handler before attempting
to close it, in order to suppress a harmless BlockingIOError warning
during loop close.

Closes: https://bugs.gentoo.org/655656
Reported-by: Helmut Jarausch  igpm.rwth-aachen.de>

 pym/portage/util/_eventloop/asyncio_event_loop.py | 10 +-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py 
b/pym/portage/util/_eventloop/asyncio_event_loop.py
index b365939b0..bf5937de8 100644
--- a/pym/portage/util/_eventloop/asyncio_event_loop.py
+++ b/pym/portage/util/_eventloop/asyncio_event_loop.py
@@ -1,6 +1,8 @@
 # Copyright 2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import signal
+
 try:
import asyncio as _real_asyncio
from asyncio.events import AbstractEventLoop as _AbstractEventLoop
@@ -31,7 +33,6 @@ class AsyncioEventLoop(_AbstractEventLoop):
self.call_at = loop.call_at
self.is_running = loop.is_running
self.is_closed = loop.is_closed
-   self.close = loop.close
self.create_future = (loop.create_future
if hasattr(loop, 'create_future') else 
self._create_future)
self.create_task = loop.create_task
@@ -75,3 +76,10 @@ class AsyncioEventLoop(_AbstractEventLoop):
@return: the internal event loop's AbstractEventLoop interface
"""
return self
+
+   def close(self):
+   # Suppress spurious error messages like the following for bug 
655656:
+   #   Exception ignored when trying to write to the signal wakeup 
fd:
+   #   BlockingIOError: [Errno 11] Resource temporarily unavailable
+   self._loop.remove_signal_handler(signal.SIGCHLD)
+   self._loop.close()

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/

2018-05-09 Thread Zac Medico

commit: 3a55ecd1f79c31f477d7bdd0b9f0e97d8a15eb9e
Author: Zac Medico  gentoo  org>
AuthorDate: Wed May  9 14:19:11 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed May  9 14:19:11 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3a55ecd1

DefaultEventLoopPolicy: test NotImplementedError due to recursion

 .../asyncio/test_policy_wrapper_recursion.py   | 29 ++
 1 file changed, 29 insertions(+)

diff --git 
a/pym/portage/tests/util/futures/asyncio/test_policy_wrapper_recursion.py 
b/pym/portage/tests/util/futures/asyncio/test_policy_wrapper_recursion.py
new file mode 100644
index 0..d3cd94b35
--- /dev/null
+++ b/pym/portage/tests/util/futures/asyncio/test_policy_wrapper_recursion.py
@@ -0,0 +1,29 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+try:
+   import asyncio
+except ImportError:
+   asyncio = None
+
+from portage.tests import TestCase
+from portage.util.futures.unix_events import DefaultEventLoopPolicy
+
+
+class PolicyWrapperRecursionTestCase(TestCase):
+   def testPolicyWrapperRecursion(self):
+   if asyncio is None:
+   self.skipTest('asyncio is not available')
+
+   initial_policy = asyncio.get_event_loop_policy()
+   if not isinstance(initial_policy, DefaultEventLoopPolicy):
+   asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
+
+   try:
+   with self.assertRaises(NotImplementedError):
+   asyncio.get_event_loop()
+
+   with self.assertRaises(NotImplementedError):
+   asyncio.get_child_watcher()
+   finally:
+   asyncio.set_event_loop_policy(initial_policy)

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/

2018-05-09 Thread Zac Medico

commit: 0c96d2d0b18036cb94d68c42783441d32af2d9d3
Author: Zac Medico  gentoo  org>
AuthorDate: Wed May  9 07:54:26 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed May  9 07:59:56 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0c96d2d0

SubprocessExecTestCase: fix unintended skipTest

The internal asyncio shim does not provide a
create_subprocess_exec attribute, so access it
directly from the real asyncio module.

Fixes 82a3cda6f1ff ("Minimize _asyncio_wrapper usage (bug 654390)")

 .../util/futures/asyncio/test_subprocess_exec.py| 21 +
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py 
b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
index 534d79c53..5a812ba6a 100644
--- a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
+++ b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
@@ -4,6 +4,11 @@
 import os
 import subprocess
 
+try:
+   from asyncio import create_subprocess_exec
+except ImportError:
+   create_subprocess_exec = None
+
 from portage.process import find_binary
 from portage.tests import TestCase
 from portage.util._eventloop.global_event_loop import global_event_loop
@@ -71,7 +76,7 @@ class SubprocessExecTestCase(TestCase):

self.assertFalse(global_event_loop().is_closed())
 
def testEcho(self):
-   if not hasattr(asyncio, 'create_subprocess_exec'):
+   if create_subprocess_exec is None:
self.skipTest('create_subprocess_exec not implemented 
for python2')
 
args_tuple = (b'hello', b'world')
@@ -91,7 +96,7 @@ class SubprocessExecTestCase(TestCase):
try:
with open(os.devnull, 'rb', 0) as devnull:
proc = loop.run_until_complete(
-   asyncio.create_subprocess_exec(
+   create_subprocess_exec(
echo_binary, *args_tuple,
stdin=devnull, 
stdout=stdout_pw, stderr=stdout_pw))
 
@@ -114,7 +119,7 @@ class SubprocessExecTestCase(TestCase):
self._run_test(test)
 
def testCat(self):
-   if not hasattr(asyncio, 'create_subprocess_exec'):
+   if create_subprocess_exec is None:
self.skipTest('create_subprocess_exec not implemented 
for python2')
 
stdin_data = b'hello world'
@@ -138,7 +143,7 @@ class SubprocessExecTestCase(TestCase):
output = None
try:
proc = loop.run_until_complete(
-   asyncio.create_subprocess_exec(
+   create_subprocess_exec(
cat_binary,
stdin=stdin_pr, stdout=stdout_pw, 
stderr=stdout_pw))
 
@@ -173,7 +178,7 @@ class SubprocessExecTestCase(TestCase):
requires an AbstractEventLoop.connect_read_pipe implementation
(and a ReadTransport implementation for it to return).
"""
-   if not hasattr(asyncio, 'create_subprocess_exec'):
+   if create_subprocess_exec is None:
self.skipTest('create_subprocess_exec not implemented 
for python2')
 
args_tuple = (b'hello', b'world')
@@ -184,7 +189,7 @@ class SubprocessExecTestCase(TestCase):
def test(loop):
with open(os.devnull, 'rb', 0) as devnull:
proc = loop.run_until_complete(
-   asyncio.create_subprocess_exec(
+   create_subprocess_exec(
echo_binary, *args_tuple,
stdin=devnull,
stdout=subprocess.PIPE, 
stderr=subprocess.STDOUT))
@@ -202,7 +207,7 @@ class SubprocessExecTestCase(TestCase):
requires an AbstractEventLoop.connect_write_pipe implementation
(and a WriteTransport implementation for it to return).
"""
-   if not hasattr(asyncio, 'create_subprocess_exec'):
+   if create_subprocess_exec is None:
self.skipTest('create_subprocess_exec not implemented 
for python2')
 
stdin_data = b'hello world'
@@ -212,7 +217,7 @@ class SubprocessExecTestCase(TestCase):
 
def test(loop):
proc = loop.run_until_complete(
-   asyncio.create_subprocess_exec(
+   create_subprocess_exec(

[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/_asyncio/, pym/portage/util/futures/

2018-05-09 Thread Zac Medico

commit: 5e628787e6f4c720680aeeb8beeac88e37988a9e
Author: Zac Medico  gentoo  org>
AuthorDate: Wed May  9 07:33:37 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed May  9 07:40:34 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5e628787

DefaultEventLoopPolicy: raise NotImplementedError, not RecursionError

Since the DefaultEventLoopPolicy wraps the underlying asyncio event
loop policy, raise NotImplementedError if the current instance is set
as the underlying event loop policy. This avoids a RecursionError
that would flood the terminal with a large stack trace.

 pym/portage/util/futures/_asyncio/__init__.py |  6 +++---
 pym/portage/util/futures/unix_events.py   | 23 ++-
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/pym/portage/util/futures/_asyncio/__init__.py 
b/pym/portage/util/futures/_asyncio/__init__.py
index 940da4762..acfd59396 100644
--- a/pym/portage/util/futures/_asyncio/__init__.py
+++ b/pym/portage/util/futures/_asyncio/__init__.py
@@ -32,7 +32,7 @@ except ImportError:
 
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
-   'portage.util.futures.unix_events:DefaultEventLoopPolicy',
+   'portage.util.futures.unix_events:_PortageEventLoopPolicy',
 )
 from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as 
_AsyncioEventLoop
 from portage.util._eventloop.global_event_loop import (
@@ -67,7 +67,7 @@ def get_event_loop_policy():
global _lock, _policy
with _lock:
if _policy is None:
-   _policy = DefaultEventLoopPolicy()
+   _policy = _PortageEventLoopPolicy()
return _policy
 
 
@@ -81,7 +81,7 @@ def set_event_loop_policy(policy):
"""
global _lock, _policy
with _lock:
-   _policy = policy or DefaultEventLoopPolicy()
+   _policy = policy or _PortageEventLoopPolicy()
 
 
 def get_event_loop():

diff --git a/pym/portage/util/futures/unix_events.py 
b/pym/portage/util/futures/unix_events.py
index 8eb369f8b..3381eaa7d 100644
--- a/pym/portage/util/futures/unix_events.py
+++ b/pym/portage/util/futures/unix_events.py
@@ -681,4 +681,25 @@ class 
_PortageEventLoopPolicy(events.AbstractEventLoopPolicy):
return _global_event_loop()._asyncio_child_watcher
 
 
-DefaultEventLoopPolicy = _PortageEventLoopPolicy
+class _AsyncioEventLoopPolicy(_PortageEventLoopPolicy):
+   """
+   A subclass of _PortageEventLoopPolicy which raises
+   NotImplementedError if it is set as the real asyncio event loop
+   policy, since this class is intended to *wrap* the real asyncio
+   event loop policy.
+   """
+   def _check_recursion(self):
+   if _real_asyncio.get_event_loop_policy() is self:
+   raise NotImplementedError('this class is only a 
wrapper')
+
+   def get_event_loop(self):
+   self._check_recursion()
+   return super(_AsyncioEventLoopPolicy, self).get_event_loop()
+
+   def get_child_watcher(self):
+   self._check_recursion()
+   return super(_AsyncioEventLoopPolicy, self).get_child_watcher()
+
+
+DefaultEventLoopPolicy = (_AsyncioEventLoopPolicy if _asyncio_enabled
+   else _PortageEventLoopPolicy)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/_asyncio/, pym/portage/util/futures/

2018-05-08 Thread Zac Medico

commit: 920b90fd0883dbd36f0290d08c9af49a208c2950
Author: Zac Medico  gentoo  org>
AuthorDate: Wed May  9 04:59:59 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed May  9 04:59:59 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=920b90fd

_wrap_loop: default to global_event_loop behavior

The default loop returned by _wrap_loop should be consistent
with global_event_loop, in order to avoid accidental registration
of callbacks with a loop that is not intended to run.

Fixes 96cc07326391 ("global_event_loop: use asyncio event loop (bug 654390)")

 pym/portage/util/futures/_asyncio/__init__.py | 14 ++
 pym/portage/util/futures/unix_events.py   | 40 +--
 2 files changed, 10 insertions(+), 44 deletions(-)

diff --git a/pym/portage/util/futures/_asyncio/__init__.py 
b/pym/portage/util/futures/_asyncio/__init__.py
index 1273afa02..940da4762 100644
--- a/pym/portage/util/futures/_asyncio/__init__.py
+++ b/pym/portage/util/futures/_asyncio/__init__.py
@@ -35,7 +35,10 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures.unix_events:DefaultEventLoopPolicy',
 )
 from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as 
_AsyncioEventLoop
-from portage.util._eventloop.global_event_loop import _asyncio_enabled
+from portage.util._eventloop.global_event_loop import (
+   _asyncio_enabled,
+   global_event_loop as _global_event_loop,
+)
 from portage.util.futures.futures import (
CancelledError,
Future,
@@ -168,14 +171,15 @@ def _wrap_loop(loop=None):
@rtype: asyncio.AbstractEventLoop (or compatible)
@return: event loop
"""
-   return loop or get_event_loop()
+   return loop or _global_event_loop()
 
 
 if _asyncio_enabled:
-   get_event_loop_policy = _real_asyncio.get_event_loop_policy
-   set_event_loop_policy = _real_asyncio.set_event_loop_policy
+   # The default loop returned by _wrap_loop should be consistent
+   # with global_event_loop, in order to avoid accidental registration
+   # of callbacks with a loop that is not intended to run.
 
def _wrap_loop(loop=None):
-   loop = loop or get_event_loop()
+   loop = loop or _global_event_loop()
return (loop if hasattr(loop, '_asyncio_wrapper')
else _AsyncioEventLoop(loop=loop))

diff --git a/pym/portage/util/futures/unix_events.py 
b/pym/portage/util/futures/unix_events.py
index ce520db00..8eb369f8b 100644
--- a/pym/portage/util/futures/unix_events.py
+++ b/pym/portage/util/futures/unix_events.py
@@ -681,42 +681,4 @@ class 
_PortageEventLoopPolicy(events.AbstractEventLoopPolicy):
return _global_event_loop()._asyncio_child_watcher
 
 
-class _AsyncioEventLoopPolicy(_PortageEventLoopPolicy):
-   """
-   Implementation of asyncio.AbstractEventLoopPolicy based on asyncio's
-   event loop. This supports running event loops in forks,
-   which is not supported by the default asyncio event loop policy,
-   see https://bugs.python.org/issue22087 and also
-   https://bugs.python.org/issue29703 which affects pypy3-5.10.1.
-   """
-   _MAIN_PID = os.getpid()
-
-   def __init__(self):
-   super(_AsyncioEventLoopPolicy, self).__init__()
-   self._default_policy = _real_asyncio.DefaultEventLoopPolicy()
-
-   def get_event_loop(self):
-   """
-   Get the event loop for the current context.
-
-   Returns an event loop object implementing the AbstractEventLoop
-   interface.
-
-   @rtype: asyncio.AbstractEventLoop (or compatible)
-   @return: the current event loop policy
-   """
-   if os.getpid() == self._MAIN_PID:
-   return self._default_policy.get_event_loop()
-   else:
-   return super(_AsyncioEventLoopPolicy, 
self).get_event_loop()
-
-   def get_child_watcher(self):
-   """Get the watcher for child processes."""
-   if os.getpid() == self._MAIN_PID:
-   return self._default_policy.get_child_watcher()
-   else:
-   return super(_AsyncioEventLoopPolicy, 
self).get_child_watcher()
-
-
-DefaultEventLoopPolicy = (_AsyncioEventLoopPolicy if _asyncio_enabled
-   else _PortageEventLoopPolicy)
+DefaultEventLoopPolicy = _PortageEventLoopPolicy

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, pym/portage/util/_eventloop/, ...

2018-05-08 Thread Zac Medico

commit: 96cc073263910e7c1b0f48cc08a4db4b56ffe408
Author: Zac Medico  gentoo  org>
AuthorDate: Tue May  1 08:15:39 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue May  8 16:42:03 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=96cc0732

global_event_loop: use asyncio event loop (bug 654390)

For python3.4 and later, in the main process, replace portage's
internal event loop with the standard library's asyncio event
loop. Continue to use portage's internal event loop in subprocesses,
since asyncio's event loop is not guaranteed to work well in
subprocesses (see upstream python issues 22087 and 29703).

An _AsyncioEventLoopPolicy class, derived from _PortageEventLoopPolicy,
is needed for some unit tests that modify asyncio's event loop policy.
This policy is not needed for anything other than unit testing. Portage
uses asyncio's default event loop policy, and API consumers are free to
use any desired event loop policy.

Portage's asynchronous functions that accept a 'loop' parameter will
work with any compatible asyncio.AbstractEventLoop implementation, since
an internal _wrap_loop function automatically adapts the loop for
internal use.

Bug: https://bugs.gentoo.org/654390
Reviewed-by: Brian Dolbec  gentoo.org>
Reviewed-by: Alec Warner  gentoo.org>

 pym/portage/util/_async/SchedulerInterface.py |  3 +
 pym/portage/util/_eventloop/asyncio_event_loop.py | 77 +++
 pym/portage/util/_eventloop/global_event_loop.py  |  5 +-
 pym/portage/util/futures/_asyncio/__init__.py | 17 +
 pym/portage/util/futures/unix_events.py   | 43 -
 5 files changed, 143 insertions(+), 2 deletions(-)

diff --git a/pym/portage/util/_async/SchedulerInterface.py 
b/pym/portage/util/_async/SchedulerInterface.py
index f1a3e9b0b..ec6417da1 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -33,6 +33,9 @@ class SchedulerInterface(SlotObject):
"time",
 
"_asyncio_child_watcher",
+   # This attribute it used by _wrap_loop to detect if the
+   # loop already has a suitable wrapper.
+   "_asyncio_wrapper",
)
 
__slots__ = _event_loop_attrs + ("_event_loop", "_is_background")

diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py 
b/pym/portage/util/_eventloop/asyncio_event_loop.py
new file mode 100644
index 0..b365939b0
--- /dev/null
+++ b/pym/portage/util/_eventloop/asyncio_event_loop.py
@@ -0,0 +1,77 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+try:
+   import asyncio as _real_asyncio
+   from asyncio.events import AbstractEventLoop as _AbstractEventLoop
+except ImportError:
+   # Allow ImportModulesTestCase to succeed.
+   _real_asyncio = None
+   _AbstractEventLoop = object
+
+
+class AsyncioEventLoop(_AbstractEventLoop):
+   """
+   Implementation of asyncio.AbstractEventLoop which wraps asyncio's
+   event loop and is minimally compatible with _PortageEventLoop.
+   """
+
+   # Use portage's internal event loop in subprocesses, as a workaround
+   # for https://bugs.python.org/issue22087, and also
+   # https://bugs.python.org/issue29703 which affects pypy3-5.10.1.
+   supports_multiprocessing = False
+
+   def __init__(self, loop=None):
+   loop = loop or _real_asyncio.get_event_loop()
+   self._loop = loop
+   self.run_until_complete = loop.run_until_complete
+   self.call_soon = loop.call_soon
+   self.call_soon_threadsafe = loop.call_soon_threadsafe
+   self.call_later = loop.call_later
+   self.call_at = loop.call_at
+   self.is_running = loop.is_running
+   self.is_closed = loop.is_closed
+   self.close = loop.close
+   self.create_future = (loop.create_future
+   if hasattr(loop, 'create_future') else 
self._create_future)
+   self.create_task = loop.create_task
+   self.add_reader = loop.add_reader
+   self.remove_reader = loop.remove_reader
+   self.add_writer = loop.add_writer
+   self.remove_writer = loop.remove_writer
+   self.run_in_executor = loop.run_in_executor
+   self.time = loop.time
+   self.default_exception_handler = loop.default_exception_handler
+   self.call_exception_handler = loop.call_exception_handler
+   self.set_debug = loop.set_debug
+   self.get_debug = loop.get_debug
+
+   def _create_future(self):
+   """
+   Provide AbstractEventLoop.create_future() for python3.4.
+   """
+   return _real_asyncio.Future(loop=self._loop)
+
+   @property
+   def _asyncio_child_watcher(self):
+

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

2018-05-07 Thread Zac Medico

commit: 505707a335683330ac69b1c7f428e8a1c66d0b2e
Author: Zac Medico  gentoo  org>
AuthorDate: Mon May  7 08:14:19 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon May  7 08:17:18 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=505707a3

rsync: explicitly use ForkExecutor for key refresh retry (bug 654390)

The ThreadPoolExecutor that asyncio uses by default does not support
cancellation of tasks, therefore use ForkExecutor for task cancellation
support, in order to enforce timeouts.

Bug: https://bugs.gentoo.org/654390

 pym/portage/sync/modules/rsync/rsync.py | 14 ++
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 070798a53..382a1eaae 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -26,6 +26,7 @@ from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, 
RSYNC_PACKAGE_ATOM
 from portage.util._eventloop.global_event_loop import global_event_loop
 from portage.util import writemsg, writemsg_stdout
 from portage.util.futures import asyncio
+from portage.util.futures.executor.fork import ForkExecutor
 from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import NewBase
@@ -170,11 +171,16 @@ class RsyncSync(NewBase):

level=logging.ERROR, noiselevel=-1)
raise # retry
 
+   # The ThreadPoolExecutor that 
asyncio uses by default
+   # does not support cancellation 
of tasks, therefore
+   # use ForkExecutor for task 
cancellation support, in
+   # order to enforce timeouts.
loop = global_event_loop()
-   func_coroutine = 
functools.partial(loop.run_in_executor,
-   None, 
noisy_refresh_keys)
-   decorated_func = 
retry_decorator(func_coroutine, loop=loop)
-   
loop.run_until_complete(decorated_func())
+   with ForkExecutor(loop=loop) as 
executor:
+   func_coroutine = 
functools.partial(loop.run_in_executor,
+   executor, 
noisy_refresh_keys)
+   decorated_func = 
retry_decorator(func_coroutine, loop=loop)
+   
loop.run_until_complete(decorated_func())
out.eend(0)
except (GematoException, asyncio.TimeoutError) 
as e:
writemsg_level("!!! Manifest 
verification impossible due to keyring problem:\n%s\n"

[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/, pym/portage/tests/util/futures/, ...

2018-05-07 Thread Zac Medico

commit: b3b15c451cc21a2c53638f0eacc8396e395dcab3
Author: Zac Medico  gentoo  org>
AuthorDate: Mon May  7 06:24:22 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon May  7 06:28:47 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b3b15c45

retry: add loop parameter during decoration

 pym/portage/sync/modules/rsync/rsync.py  |  2 +-
 pym/portage/tests/util/futures/test_retry.py | 16 
 pym/portage/util/futures/retry.py|  6 +++---
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 1b8941ff6..070798a53 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -173,7 +173,7 @@ class RsyncSync(NewBase):
loop = global_event_loop()
func_coroutine = 
functools.partial(loop.run_in_executor,
None, 
noisy_refresh_keys)
-   decorated_func = 
retry_decorator(func_coroutine)
+   decorated_func = 
retry_decorator(func_coroutine, loop=loop)

loop.run_until_complete(decorated_func())
out.eend(0)
except (GematoException, asyncio.TimeoutError) 
as e:

diff --git a/pym/portage/tests/util/futures/test_retry.py 
b/pym/portage/tests/util/futures/test_retry.py
index 16ecccbc7..7a1e76280 100644
--- a/pym/portage/tests/util/futures/test_retry.py
+++ b/pym/portage/tests/util/futures/test_retry.py
@@ -85,7 +85,7 @@ class RetryTestCase(TestCase):
func_coroutine = self._wrap_coroutine_func(SucceedLater(1))
decorator = retry(try_max=,
delay_func=RandomExponentialBackoff(multiplier=0.1, 
base=2))
-   decorated_func = decorator(func_coroutine)
+   decorated_func = decorator(func_coroutine, loop=loop)
result = loop.run_until_complete(decorated_func())
self.assertEqual(result, 'success')
 
@@ -94,7 +94,7 @@ class RetryTestCase(TestCase):
func_coroutine = self._wrap_coroutine_func(SucceedNever())
decorator = retry(try_max=4, try_timeout=None,
delay_func=RandomExponentialBackoff(multiplier=0.1, 
base=2))
-   decorated_func = decorator(func_coroutine)
+   decorated_func = decorator(func_coroutine, loop=loop)
done, pending = 
loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(isinstance(done.pop().exception().__cause__, 
SucceedNeverException))
@@ -104,7 +104,7 @@ class RetryTestCase(TestCase):
func_coroutine = self._wrap_coroutine_func(SucceedNever())
decorator = retry(reraise=True, try_max=4, try_timeout=None,
delay_func=RandomExponentialBackoff(multiplier=0.1, 
base=2))
-   decorated_func = decorator(func_coroutine)
+   decorated_func = decorator(func_coroutine, loop=loop)
done, pending = 
loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(isinstance(done.pop().exception(), 
SucceedNeverException))
@@ -114,7 +114,7 @@ class RetryTestCase(TestCase):
func_coroutine = self._wrap_coroutine_func(HangForever())
decorator = retry(try_max=2, try_timeout=0.1,
delay_func=RandomExponentialBackoff(multiplier=0.1, 
base=2))
-   decorated_func = decorator(func_coroutine)
+   decorated_func = decorator(func_coroutine, loop=loop)
done, pending = 
loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(isinstance(done.pop().exception().__cause__, 
asyncio.TimeoutError))
@@ -124,7 +124,7 @@ class RetryTestCase(TestCase):
func_coroutine = self._wrap_coroutine_func(HangForever())
decorator = retry(reraise=True, try_max=2, try_timeout=0.1,
delay_func=RandomExponentialBackoff(multiplier=0.1, 
base=2))
-   decorated_func = decorator(func_coroutine)
+   decorated_func = decorator(func_coroutine, loop=loop)
done, pending = 
loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(isinstance(done.pop().exception(), 
asyncio.TimeoutError))
@@ -134,7 +134,7 @@ class RetryTestCase(TestCase):
func_coroutine =

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/

2018-05-06 Thread Zac Medico

commit: 299a9a40536209c4c2664857b5aecb018fa0a6f6
Author: Zac Medico  gentoo  org>
AuthorDate: Sun May  6 23:40:33 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon May  7 00:19:54 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=299a9a40

TestCase: handle SkipTest during setUp method

 pym/portage/tests/__init__.py | 20 ++--
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/pym/portage/tests/__init__.py b/pym/portage/tests/__init__.py
index bf681c47e..e149b5c0c 100644
--- a/pym/portage/tests/__init__.py
+++ b/pym/portage/tests/__init__.py
@@ -207,18 +207,18 @@ class TestCase(unittest.TestCase):
result.startTest(self)
testMethod = getattr(self, self._testMethodName)
try:
-   try:
-   self.setUp()
-   except SystemExit:
-   raise
-   except KeyboardInterrupt:
-   raise
-   except:
-   result.addError(self, sys.exc_info())
-   return
-
ok = False
try:
+   try:
+   self.setUp()
+   except KeyboardInterrupt:
+   raise
+   except SkipTest:
+   raise
+   except Exception:
+   result.addError(self, sys.exc_info())
+   return
+
testMethod()
ok = True
except SkipTest as e:

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/

2018-05-06 Thread Zac Medico

commit: fefc6f1feb57503ce4826de1e405c12ef761c0ce
Author: Zac Medico  gentoo  org>
AuthorDate: Sun May  6 23:41:15 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon May  7 00:19:54 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fefc6f1f

RetryTestCase: test ThreadPoolExecutor and ForkExecutor (bug 654390)

ThreadPoolExecutor is the default asyncio event loop's default executor,
so explicitly test it. Also explicitly test ForkExecutor, since it is
more useful in cases where tasks may need to be forcefully cancelled.

Bug: https://bugs.gentoo.org/654390

 pym/portage/tests/util/futures/test_retry.py | 37 ++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/pym/portage/tests/util/futures/test_retry.py 
b/pym/portage/tests/util/futures/test_retry.py
index baf293d56..16ecccbc7 100644
--- a/pym/portage/tests/util/futures/test_retry.py
+++ b/pym/portage/tests/util/futures/test_retry.py
@@ -1,16 +1,24 @@
 # Copyright 2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+try:
+   from concurrent.futures import ThreadPoolExecutor
+except ImportError:
+   ThreadPoolExecutor = None
+
 try:
import threading
 except ImportError:
import dummy_threading as threading
 
+import sys
+
 from portage.tests import TestCase
 from portage.util._eventloop.global_event_loop import global_event_loop
 from portage.util.backoff import RandomExponentialBackoff
 from portage.util.futures import asyncio
 from portage.util.futures.retry import retry
+from portage.util.futures.executor.fork import ForkExecutor
 from portage.util.monotonic import monotonic
 
 
@@ -155,13 +163,31 @@ class RetryTestCase(TestCase):
self.assertTrue(isinstance(done.pop().exception().__cause__, 
asyncio.TimeoutError))
 
 
-class RetryExecutorTestCase(RetryTestCase):
+class RetryForkExecutorTestCase(RetryTestCase):
"""
Wrap each coroutine function with AbstractEventLoop.run_in_executor,
in order to test the event loop's default executor. The executor
may use either a thread or a subprocess, and either case is
automatically detected and handled.
"""
+   def __init__(self, *pargs, **kwargs):
+   super(RetryForkExecutorTestCase, self).__init__(*pargs, 
**kwargs)
+   self._executor = None
+
+   def _setUpExecutor(self):
+   self._executor = ForkExecutor()
+
+   def _tearDownExecutor(self):
+   if self._executor is not None:
+   self._executor.shutdown(wait=True)
+   self._executor = None
+
+   def setUp(self):
+   self._setUpExecutor()
+
+   def tearDown(self):
+   self._tearDownExecutor()
+
def _wrap_coroutine_func(self, coroutine_func):
parent_loop = global_event_loop()
 
@@ -191,7 +217,7 @@ class RetryExecutorTestCase(RetryTestCase):
def execute_wrapper():
kill_switch = parent_loop.create_future()
parent_future = asyncio.ensure_future(
-   parent_loop.run_in_executor(None, wrapper, 
kill_switch),
+   parent_loop.run_in_executor(self._executor, 
wrapper, kill_switch),
loop=parent_loop)
parent_future.add_done_callback(
lambda parent_future: None if kill_switch.done()
@@ -199,3 +225,10 @@ class RetryExecutorTestCase(RetryTestCase):
return parent_future
 
return execute_wrapper
+
+
+class RetryThreadExecutorTestCase(RetryForkExecutorTestCase):
+   def _setUpExecutor(self):
+   if sys.version_info.major < 3:
+   self.skipTest('ThreadPoolExecutor not supported for 
python2')
+   self._executor = ThreadPoolExecutor(max_workers=1)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/

2018-05-06 Thread Zac Medico

commit: a1360e017b7ee8156ad4ad850e2f8ea40228ca1a
Author: Zac Medico  gentoo  org>
AuthorDate: Mon May  7 00:06:07 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon May  7 00:19:54 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a1360e01

EventLoop.run_in_executor: use asyncio.wrap_future

Since executors from the concurrent.futures package return
concurrent.futures.Future, it's necessary to wrap them.

 pym/portage/util/_eventloop/EventLoop.py | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/pym/portage/util/_eventloop/EventLoop.py 
b/pym/portage/util/_eventloop/EventLoop.py
index fc7380b03..de0795224 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -13,6 +13,11 @@ import signal
 import sys
 import traceback
 
+try:
+   import asyncio as _real_asyncio
+except ImportError:
+   _real_asyncio = None
+
 try:
import fcntl
 except ImportError:
@@ -937,7 +942,11 @@ class EventLoop(object):
if executor is None:
executor = ForkExecutor(loop=self)
self._default_executor = executor
-   return executor.submit(func, *args)
+   future = executor.submit(func, *args)
+   if _real_asyncio is not None:
+   future = _real_asyncio.wrap_future(future,
+   loop=self._asyncio_wrapper)
+   return future
 
def is_running(self):
"""Return whether the event loop is currently running."""

[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/_asyncio/, pym/portage/tests/util/futures/asyncio/, ...

2018-05-06 Thread Zac Medico

commit: 85ac23b7c0c58cef72d22281d66d086521c01e3e
Author: Zac Medico  gentoo  org>
AuthorDate: Sun May  6 11:05:03 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun May  6 11:41:45 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=85ac23b7

asyncio: add _wrap_loop helper (bug 654390)

In order to deal with asyncio event loop compatibility issues, add
a _wrap_loop helper. For example, since python3.4 does not have the
AbstractEventLoop.create_future() method, this helper function can
be used to add a wrapper that implements the create_future method
for python3.4.

Bug: https://bugs.gentoo.org/654390

 pym/portage/dbapi/porttree.py | 12 ++--
 .../tests/util/futures/asyncio/test_child_watcher.py  |  2 +-
 .../util/futures/asyncio/test_event_loop_in_fork.py   |  8 
 .../tests/util/futures/asyncio/test_pipe_closed.py|  4 ++--
 .../util/futures/asyncio/test_run_until_complete.py   |  2 +-
 .../util/futures/asyncio/test_subprocess_exec.py  |  4 ++--
 pym/portage/util/futures/_asyncio/__init__.py | 19 ++-
 pym/portage/util/futures/_asyncio/tasks.py|  7 +--
 pym/portage/util/futures/executor/fork.py |  4 ++--
 pym/portage/util/futures/iter_completed.py|  7 +++
 pym/portage/util/futures/retry.py |  3 +--
 11 files changed, 45 insertions(+), 27 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 801b5658a..3e36024ff 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -36,7 +36,7 @@ from portage import _encodings
 from portage import _unicode_encode
 from portage import OrderedDict
 from portage.util._eventloop.EventLoop import EventLoop
-from portage.util._eventloop.global_event_loop import global_event_loop
+from portage.util.futures import asyncio
 from portage.util.futures.iter_completed import iter_gather
 from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
 
@@ -325,8 +325,8 @@ class portdbapi(dbapi):
@property
def _event_loop(self):
if portage._internal_caller:
-   # For internal portage usage, the global_event_loop is 
safe.
-   return global_event_loop()
+   # For internal portage usage, asyncio._wrap_loop() is 
safe.
+   return asyncio._wrap_loop()
else:
# For external API consumers, use a local EventLoop, 
since
# we don't want to assume that it's safe to override the
@@ -611,7 +611,7 @@ class portdbapi(dbapi):
# to simultaneous instantiation of multiple event loops here.
# Callers of this method certainly want the same event loop to
# be used for all calls.
-   loop = loop or global_event_loop()
+   loop = asyncio._wrap_loop(loop)
future = loop.create_future()
cache_me = False
if myrepo is not None:
@@ -751,7 +751,7 @@ class portdbapi(dbapi):
a set of alternative URIs.
@rtype: asyncio.Future (or compatible)
"""
-   loop = loop or global_event_loop()
+   loop = asyncio._wrap_loop(loop)
result = loop.create_future()
 
def aux_get_done(aux_get_future):
@@ -1419,7 +1419,7 @@ def _async_manifest_fetchlist(portdb, repo_config, cp, 
cpv_list=None,
@return: a Future resulting in a Mapping compatible with FetchlistDict
@rtype: asyncio.Future (or compatible)
"""
-   loop = loop or global_event_loop()
+   loop = asyncio._wrap_loop(loop)
result = loop.create_future()
cpv_list = (portdb.cp_list(cp, mytree=repo_config.location)
if cpv_list is None else cpv_list)

diff --git a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py 
b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
index dca01be56..8ef497544 100644
--- a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
+++ b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
@@ -28,7 +28,7 @@ class ChildWatcherTestCase(TestCase):
 
args_tuple = ('hello', 'world')
 
-   loop = asyncio.get_event_loop()
+   loop = asyncio._wrap_loop()
future = loop.create_future()
 
def callback(pid, returncode, *args):

diff --git a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py 
b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py
index 7868d792a..19588bf3a 100644
--- a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py
+++ b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py
@@ -11,14 +11,14 @@ from portage.util.futures.unix_events import

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/

2018-05-05 Thread Zac Medico

commit: 87aeab1a62cc6fa1d48354a42ec4fa787dbe9603
Author: Zac Medico  gentoo  org>
AuthorDate: Sun May  6 01:19:08 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun May  6 01:26:50 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=87aeab1a

WriterPipeClosedTestCase: retry filling pipe

This should suppress spurious writer callback observed
twice for pypy in travis.

See: https://travis-ci.org/gentoo/portage/jobs/375411936
See: https://travis-ci.org/gentoo/portage/jobs/373734825

 .../tests/util/futures/asyncio/test_pipe_closed.py | 39 +-
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py 
b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
index 5398ca35c..e63829888 100644
--- a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
+++ b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
@@ -105,25 +105,32 @@ class WriterPipeClosedTestCase(_PipeClosedTestCase, 
TestCase):
 
writer_callback.called = loop.create_future()
_set_nonblocking(write_end.fileno())
+   loop.add_writer(write_end.fileno(), writer_callback)
 
-   # Fill up the pipe, so that no writer callbacks should 
be
-   # received until the state has changed.
-   while True:
-   try:
-   os.write(write_end.fileno(), 512 * b'0')
-   except EnvironmentError as e:
-   if e.errno != errno.EAGAIN:
-   raise
+   # With pypy we've seen intermittent spurious writer 
callbacks
+   # here, so retry until the correct state is achieved.
+   tries = 10
+   while tries:
+   tries -= 1
+
+   # Fill up the pipe, so that no writer callbacks 
should be
+   # received until the state has changed.
+   while True:
+   try:
+   os.write(write_end.fileno(), 
512 * b'0')
+   except EnvironmentError as e:
+   if e.errno != errno.EAGAIN:
+   raise
+   break
+
+   # Allow the loop to check for IO events, and 
assert
+   # that our future is still not done.
+   loop.run_until_complete(asyncio.sleep(0, 
loop=loop))
+   if writer_callback.called.done():
+   writer_callback.called = 
loop.create_future()
+   else:
break
 
-   # We've seen at least one spurious writer callback when
-   # this was registered before the pipe was filled, so
-   # register it afterwards.
-   loop.add_writer(write_end.fileno(), writer_callback)
-
-   # Allow the loop to check for IO events, and assert
-   # that our future is still not done.
-   loop.run_until_complete(asyncio.sleep(0, loop=loop))
self.assertFalse(writer_callback.called.done())
 
# Demonstrate that the callback is called afer the

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/

2018-05-05 Thread Zac Medico

commit: 5a5ed99cb5a6e8913df2e9ca29b4b4d5c179c20f
Author: Zac Medico  gentoo  org>
AuthorDate: Sat May  5 23:04:10 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun May  6 00:35:44 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5a5ed99c

RetryTestCase: support ThreadPoolExecutor (bug 654390)

In order to support the default asyncio event loop's
ThreadPoolExecutor, use a threading.Event instance to
support cancellation of tasks.

Bug: https://bugs.gentoo.org/654390

 pym/portage/tests/util/futures/test_retry.py | 96 +---
 1 file changed, 74 insertions(+), 22 deletions(-)

diff --git a/pym/portage/tests/util/futures/test_retry.py 
b/pym/portage/tests/util/futures/test_retry.py
index cdca7d294..781eac9a1 100644
--- a/pym/portage/tests/util/futures/test_retry.py
+++ b/pym/portage/tests/util/futures/test_retry.py
@@ -1,8 +1,6 @@
 # Copyright 2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
-import functools
-
 try:
import threading
 except ImportError:
@@ -28,10 +26,17 @@ class SucceedLater(object):
self._succeed_time = monotonic() + duration
 
def __call__(self):
+   loop = global_event_loop()
+   result = loop.create_future()
remaining = self._succeed_time - monotonic()
if remaining > 0:
-   raise SucceedLaterException('time until success: {} 
seconds'.format(remaining))
-   return 'success'
+   loop.call_soon_threadsafe(lambda: None if result.done() 
else
+   result.set_exception(SucceedLaterException(
+   'time until success: {} 
seconds'.format(remaining
+   else:
+   loop.call_soon_threadsafe(lambda: None if result.done() 
else
+   result.set_result('success'))
+   return result
 
 
 class SucceedNeverException(Exception):
@@ -43,7 +48,11 @@ class SucceedNever(object):
A callable object that never succeeds.
"""
def __call__(self):
-   raise SucceedNeverException('expected failure')
+   loop = global_event_loop()
+   result = loop.create_future()
+   loop.call_soon_threadsafe(lambda: None if result.done() else
+   result.set_exception(SucceedNeverException('expected 
failure')))
+   return result
 
 
 class HangForever(object):
@@ -51,14 +60,21 @@ class HangForever(object):
A callable object that sleeps forever.
"""
def __call__(self):
-   threading.Event().wait()
+   return global_event_loop().create_future()
 
 
 class RetryTestCase(TestCase):
+
+   def _wrap_coroutine_func(self, coroutine_func):
+   """
+   Derived classes may override this method in order to implement
+   alternative forms of execution.
+   """
+   return coroutine_func
+
def testSucceedLater(self):
loop = global_event_loop()
-   func = SucceedLater(1)
-   func_coroutine = functools.partial(loop.run_in_executor, None, 
func)
+   func_coroutine = self._wrap_coroutine_func(SucceedLater(1))
decorator = retry(try_max=,
delay_func=RandomExponentialBackoff(multiplier=0.1, 
base=2))
decorated_func = decorator(func_coroutine)
@@ -67,8 +83,7 @@ class RetryTestCase(TestCase):
 
def testSucceedNever(self):
loop = global_event_loop()
-   func = SucceedNever()
-   func_coroutine = functools.partial(loop.run_in_executor, None, 
func)
+   func_coroutine = self._wrap_coroutine_func(SucceedNever())
decorator = retry(try_max=4, try_timeout=None,
delay_func=RandomExponentialBackoff(multiplier=0.1, 
base=2))
decorated_func = decorator(func_coroutine)
@@ -78,8 +93,7 @@ class RetryTestCase(TestCase):
 
def testSucceedNeverReraise(self):
loop = global_event_loop()
-   func = SucceedNever()
-   func_coroutine = functools.partial(loop.run_in_executor, None, 
func)
+   func_coroutine = self._wrap_coroutine_func(SucceedNever())
decorator = retry(reraise=True, try_max=4, try_timeout=None,
delay_func=RandomExponentialBackoff(multiplier=0.1, 
base=2))
decorated_func = decorator(func_coroutine)
@@ -89,8 +103,7 @@ class RetryTestCase(TestCase):
 
def testHangForever(self):
loop = global_event_loop()
-   func = HangForever()
-   func_coroutine = functools.partial(loop.run_in_executor, None, 
func)
+   func_coroutine = self._wrap_coroutine_func(HangForever())

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/, pym/portage/util/_async/, ...

2018-05-05 Thread Zac Medico

commit: 82a3cda6f1ffbbbd75c962d610c7eb4a04a865d9
Author: Zac Medico  gentoo  org>
AuthorDate: Sat May  5 21:00:51 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat May  5 21:00:51 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=82a3cda6

Minimize _asyncio_wrapper usage (bug 654390)

Since migration to asyncio compatible APIs is now complete,
minimize _asyncio_wrapper usage. Also remove _asyncio_wrapper from
SchedulerInterface, since there are no more consumers.

Bug: https://bugs.gentoo.org/654390

 pym/_emerge/AbstractPollTask.py  |  2 +-
 pym/portage/_emirrordist/FetchIterator.py|  1 -
 pym/portage/dbapi/porttree.py|  3 ---
 .../tests/util/futures/asyncio/test_subprocess_exec.py   |  1 -
 pym/portage/tests/util/futures/test_iter_completed.py|  4 ++--
 pym/portage/tests/util/futures/test_retry.py | 16 
 pym/portage/util/_async/SchedulerInterface.py|  1 -
 pym/portage/util/futures/__init__.py |  5 +
 pym/portage/util/futures/_asyncio/tasks.py   |  1 -
 pym/portage/util/futures/executor/fork.py|  3 +--
 pym/portage/util/futures/futures.py  |  2 +-
 pym/portage/util/futures/iter_completed.py   |  3 ---
 12 files changed, 14 insertions(+), 28 deletions(-)

diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py
index dff4b3efd..f898aa708 100644
--- a/pym/_emerge/AbstractPollTask.py
+++ b/pym/_emerge/AbstractPollTask.py
@@ -109,7 +109,7 @@ class AbstractPollTask(AsynchronousTask):
self._registered = False
 
def _wait_loop(self, timeout=None):
-   loop = getattr(self.scheduler, '_asyncio_wrapper', 
self.scheduler)
+   loop = self.scheduler
tasks = [self.async_wait()]
if timeout is not None:
tasks.append(asyncio.ensure_future(

diff --git a/pym/portage/_emirrordist/FetchIterator.py 
b/pym/portage/_emirrordist/FetchIterator.py
index 366453c12..04d4da62b 100644
--- a/pym/portage/_emirrordist/FetchIterator.py
+++ b/pym/portage/_emirrordist/FetchIterator.py
@@ -125,7 +125,6 @@ def _async_fetch_tasks(config, hash_filter, repo_config, 
digests_future, cpv,
instances for each of the files referenced by an ebuild.
@rtype: asyncio.Future (or compatible)
"""
-   loop = getattr(loop, '_asyncio_wrapper', loop)
result = loop.create_future()
fetch_tasks = []
 

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 6c38232bb..801b5658a 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -612,7 +612,6 @@ class portdbapi(dbapi):
# Callers of this method certainly want the same event loop to
# be used for all calls.
loop = loop or global_event_loop()
-   loop = getattr(loop, '_asyncio_wrapper', loop)
future = loop.create_future()
cache_me = False
if myrepo is not None:
@@ -753,7 +752,6 @@ class portdbapi(dbapi):
@rtype: asyncio.Future (or compatible)
"""
loop = loop or global_event_loop()
-   loop = getattr(loop, '_asyncio_wrapper', loop)
result = loop.create_future()
 
def aux_get_done(aux_get_future):
@@ -1422,7 +1420,6 @@ def _async_manifest_fetchlist(portdb, repo_config, cp, 
cpv_list=None,
@rtype: asyncio.Future (or compatible)
"""
loop = loop or global_event_loop()
-   loop = getattr(loop, '_asyncio_wrapper', loop)
result = loop.create_future()
cpv_list = (portdb.cp_list(cp, mytree=repo_config.location)
if cpv_list is None else cpv_list)

diff --git a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py 
b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
index be103a9e0..98983941d 100644
--- a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
+++ b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
@@ -24,7 +24,6 @@ def reader(input_file, loop=None):
@rtype: asyncio.Future (or compatible)
"""
loop = loop or asyncio.get_event_loop()
-   loop = getattr(loop, '_asyncio_wrapper', loop)
future = loop.create_future()
_Reader(future, input_file, loop)
return future

diff --git a/pym/portage/tests/util/futures/test_iter_completed.py 
b/pym/portage/tests/util/futures/test_iter_completed.py
index 90668eb02..9ab410a9e 100644
--- a/pym/portage/tests/util/futures/test_iter_completed.py
+++ b/pym/portage/tests/util/futures/test_iter_completed.py
@@ -34,7 +34,7 @@ class IterCompletedTestCase(TestCase):
# load causes the tasks to finish in an unexpected order.
self.todo = True

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/resolver/, pym/_emerge/

2018-05-04 Thread Zac Medico

commit: ce150da22e351a7ba52a6390b9cb7aa076c0c8ce
Author: Zac Medico  gentoo  org>
AuthorDate: Fri May  4 03:21:40 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Fri May  4 17:11:07 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ce150da2

depgraph.autounmask_breakage_detected: ignore REQUIRED_USE violations (bug 
654782)

When autounmask USE changes violate REQUIRED_USE, rather than
recalculate with autounmask disabled, display the autounmask USE
changes along with the REQUIRED_USE violation. Adjust unit tests
to allow for the autounmask USE changes that were previously
discarded. For the case reported in bug 654782, the output will
appear as follows:

The following keyword changes are necessary to proceed:
 (see "package.accept_keywords" in the portage(5) man page for more details)
# required by =retext-7.0.1-r1 (argument)
=app-editors/retext-7.0.1-r1 ~amd64

The following USE changes are necessary to proceed:
 (see "package.use" in the portage(5) man page for more details)
# required by app-editors/retext-7.0.1-r1::gentoo
# required by =retext-7.0.1-r1 (argument)
>=dev-python/PyQt5-5.9.2 printsupport webengine network gui widgets

!!! The ebuild selected to satisfy 
"dev-python/PyQt5[gui,network,printsupport,webengine,widgets,python_targets_python3_4(-)?,python_targets_python3_5(-)?,python_targets_python3_6(-)?,-python_single_target_python3_4(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-)]"
 has unmet requirements.
- dev-python/PyQt5-5.9.2::gentoo USE="-bluetooth -dbus -debug -declarative 
-designer -examples -gles2 -gui -help -location -multimedia -network -opengl 
-positioning -printsupport -sensors -serialport -sql -svg -testlib -webchannel 
-webengine -webkit -websockets -widgets -x11extras -xmlpatterns" 
PYTHON_TARGETS="python2_7 python3_4 python3_6 -python3_5"

  The following REQUIRED_USE flag constraints are unsatisfied:
webengine? ( widgets? ( webchannel ) )

  The above constraints are a subset of the following complete expression:
any-of ( python_targets_python2_7 python_targets_python3_4 
python_targets_python3_5 python_targets_python3_6 ) bluetooth? ( gui ) 
declarative? ( gui network ) designer? ( widgets ) help? ( gui widgets ) 
location? ( positioning ) multimedia? ( gui network ) opengl? ( gui widgets ) 
positioning? ( gui ) printsupport? ( gui widgets ) sensors? ( gui ) serialport? 
( gui ) sql? ( widgets ) svg? ( gui widgets ) testlib? ( widgets ) webchannel? 
( network ) webengine? ( network widgets? ( printsupport webchannel ) ) webkit? 
( gui network printsupport widgets ) websockets? ( network ) widgets? ( gui ) 
xmlpatterns? ( network )

(dependency required by "app-editors/retext-7.0.1-r1::gentoo" [ebuild])
(dependency required by "=retext-7.0.1-r1" [argument])

Bug: https://bugs.gentoo.org/654782
Reviewed-by: M. J. Everitt  iee.org>

 pym/_emerge/depgraph.py|  8 ++--
 pym/portage/tests/resolver/test_autounmask.py  | 48 --
 pym/portage/tests/resolver/test_slot_collisions.py | 20 +
 3 files changed, 61 insertions(+), 15 deletions(-)

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index fbd16ad98..429d8871c 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -3009,6 +3009,10 @@ class depgraph(object):
{"myparent" : dep.parent, 
"show_req_use" : pkg}))
self._dynamic_config._required_use_unsatisfied 
= True
self._dynamic_config._skip_restart = True
+   # Add pkg to digraph in order to enable 
autounmask messages
+   # for this package, which is useful when 
autounmask USE
+   # changes have violated REQUIRED_USE.
+   self._dynamic_config.digraph.add(pkg, 
dep.parent, priority=priority)
return 0
 
if not pkg.onlydeps:
@@ -9428,10 +9432,6 @@ class depgraph(object):
return self._dynamic_config._need_config_reload
 
def autounmask_breakage_detected(self):
-   # Check for REQUIRED_USE violations.
-   for changes in 
self._dynamic_config._needed_use_config_changes.values():
-   if getattr(changes, 'required_use_satisfied', None) is 
False:
-   return True
try:
for pargs, kwargs in 
self._dynamic_config._unsatisfied_deps_for_display:
self._show_unsatisfied_dep(

diff --git a/pym/portage/tests/resolver/test_autounmask.py 
b/pym/portage/tests/resolver/test_autounmask.py
index 9042349ea..809d42104 100644
--- a/pym/portage/tests/resolver/test_autounmask.py
+++ b/pym/portage/tests/resolver/test_autounmask.py
@@ -251,15 +251,42 @@ class AutounmaskTestCase(TestCase):

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/_config/

2018-05-03 Thread Zac Medico

commit: b69a756099973abbf9719717ea3726519b32ce60
Author: Zac Medico  gentoo  org>
AuthorDate: Thu May  3 23:00:35 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu May  3 23:08:47 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b69a7560

LocationsManager: fix SYSROOT normalization to handle empty SYSROOT

This issue was exposed when building portage-2.3.34 with USE="epydoc"
enabled, since epydoc triggers instantiation of portage.settings.
The empty SYSROOT change for bug 654600 (in commit
1b5110557d1dd725f7c12bbed4b7ceaaec29f2a3) triggered incorrect
normalization behavior here.

Fixes: a41dacf7926c ("Export SYSROOT and ESYSROOT in ebuild env in EAPI 7")

 pym/portage/package/ebuild/_config/LocationsManager.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/package/ebuild/_config/LocationsManager.py 
b/pym/portage/package/ebuild/_config/LocationsManager.py
index b57443ba7..f7d7209ff 100644
--- a/pym/portage/package/ebuild/_config/LocationsManager.py
+++ b/pym/portage/package/ebuild/_config/LocationsManager.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2014 Gentoo Foundation
+# Copyright 2010-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -69,7 +69,7 @@ class LocationsManager(object):
if self.sysroot is None:
self.sysroot = "/"
else:
-   self.sysroot = 
normalize_path(os.path.abspath(self.sysroot)).rstrip(os.sep) + os.sep
+   self.sysroot = 
normalize_path(os.path.abspath(self.sysroot or os.sep)).rstrip(os.sep) + os.sep
 
self.esysroot = self.sysroot.rstrip(os.sep) + self.eprefix + 
os.sep

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/_config/

2018-05-03 Thread Zac Medico

commit: cd7364fbbb048564c2714d8a08df1ecbdc9cfe6e
Author: Zac Medico  gentoo  org>
AuthorDate: Thu May  3 20:27:59 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu May  3 20:33:27 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cd7364fb

special_env_vars: add ESYSROOT and SYSROOT to environ_whitelist

Since these variables are filtered from the peristent ebuild environment
by the __filter_readonly_variables function, they have to be whitelisted
so that the config.environ() method allow them to pass through.

Fixes: a41dacf7926c ("Export SYSROOT and ESYSROOT in ebuild env in EAPI 7")
Reported-by: James Le Cuirot  gentoo.org>

 pym/portage/package/ebuild/_config/special_env_vars.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/package/ebuild/_config/special_env_vars.py 
b/pym/portage/package/ebuild/_config/special_env_vars.py
index 3323ff064..a308518af 100644
--- a/pym/portage/package/ebuild/_config/special_env_vars.py
+++ b/pym/portage/package/ebuild/_config/special_env_vars.py
@@ -46,7 +46,7 @@ environ_whitelist += [
"DISTDIR", "DOC_SYMLINKS_DIR", "EAPI", "EBUILD",
"EBUILD_FORCE_TEST",
"EBUILD_PHASE", "EBUILD_PHASE_FUNC", "ECLASSDIR", "ECLASS_DEPTH", "ED",
-   "EMERGE_FROM", "EPREFIX", "EROOT",
+   "EMERGE_FROM", "EPREFIX", "EROOT", "ESYSROOT",
"FEATURES", "FILESDIR", "HOME", "MERGE_TYPE", "NOCOLOR", "PATH",
"PKGDIR",
"PKGUSE", "PKG_LOGDIR", "PKG_TMPDIR",
@@ -77,7 +77,7 @@ environ_whitelist += [
"PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", "PORTAGE_XATTR_EXCLUDE",
"PORTDIR", "PORTDIR_OVERLAY", "PREROOTPATH", "PYTHONDONTWRITEBYTECODE",
"REPLACING_VERSIONS", "REPLACED_BY_VERSION",
-   "ROOT", "ROOTPATH", "T", "TMP", "TMPDIR",
+   "ROOT", "ROOTPATH", "SYSROOT", "T", "TMP", "TMPDIR",
"USE_EXPAND", "USE_ORDER", "WORKDIR",
"XARGS", "__PORTAGE_TEST_HARDLINK_LOCKS",
 ]

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/

2018-05-03 Thread Zac Medico

commit: 1b5110557d1dd725f7c12bbed4b7ceaaec29f2a3
Author: Zac Medico  gentoo  org>
AuthorDate: Thu May  3 00:55:48 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu May  3 18:45:07 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1b511055

config.environ: always strip slash from SYSROOT (bug 654600)

Since SYSROOT=/ interacts badly with autotools.eclass (bug 654600),
and no EAPI expects SYSROOT to have a trailing slash, always strip
the trailing slash from SYSROOT.

Bug: https://bugs.gentoo.org/654600
Fixes: a41dacf7926c ("Export SYSROOT and ESYSROOT in ebuild env in EAPI 7")
Reviewed-by: James Le Cuirot  gentoo.org>
Reviewed-by: Michał Górny  gentoo.org>
Reviewed-by: Brian Dolbec  gentoo.org>

 pym/portage/package/ebuild/config.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/package/ebuild/config.py 
b/pym/portage/package/ebuild/config.py
index f9b257b86..88acac5cc 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -2813,11 +2813,16 @@ class config(object):
mydict.pop("ECLASSDIR", None)
 
if not eapi_attrs.path_variables_end_with_trailing_slash:
-   for v in ("D", "ED", "ROOT", "EROOT", "SYSROOT", 
"ESYSROOT",
-   "BROOT"):
+   for v in ("D", "ED", "ROOT", "EROOT", "ESYSROOT", 
"BROOT"):
if v in mydict:
mydict[v] = 
mydict[v].rstrip(os.path.sep)
 
+   # Since SYSROOT=/ interacts badly with autotools.eclass (bug 
654600),
+   # and no EAPI expects SYSROOT to have a trailing slash, always 
strip
+   # the trailing slash from SYSROOT.
+   if 'SYSROOT' in mydict:
+   mydict['SYSROOT'] = mydict['SYSROOT'].rstrip(os.sep)
+
try:
builddir = mydict["PORTAGE_BUILDDIR"]
distdir = mydict["DISTDIR"]

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

2018-05-03 Thread Zac Medico

commit: 35ca4ac2f8f0de61289bf812681c21d14d2524db
Author: Zac Medico  gentoo  org>
AuthorDate: Thu May  3 06:26:19 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu May  3 06:35:45 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=35ca4ac2

rsync: silence signature info for --quiet mode (bug 654664)

Silence messages like the following for --quiet mode:

 * Using keys from /var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg
 * Refreshing keys from keyserver ...[ ok ]

 * Manifest timestamp: 2018-05-03 05:38:39 UTC
 * Valid OpenPGP signature found:
 * - primary key: DCD05B71EAB94199527F44ACDB6B8C1F96D8BF6D
 * - subkey: E1D6ABB63BFCFB4BA02FDF1CEC590EEAC9189250
 * - timestamp: 2018-05-03 05:38:39 UTC
 * Verifying /usr/portage ...[ ok ]

Bug: https://bugs.gentoo.org/654664

 pym/portage/sync/modules/rsync/rsync.py | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index b2850715e..1b8941ff6 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -66,7 +66,8 @@ class RsyncSync(NewBase):
opts = self.options.get('emerge_config').opts
self.usersync_uid = self.options.get('usersync_uid', None)
enter_invalid = '--ask-enter-invalid' in opts
-   out = portage.output.EOutput()
+   quiet = '--quiet' in opts
+   out = portage.output.EOutput(quiet=quiet)
syncuri = self.repo.sync_uri
if self.repo.module_specific_options.get(
'sync-rsync-vcs-ignore', 'false').lower() == 'true':
@@ -385,10 +386,12 @@ class RsyncSync(NewBase):
raise 
RuntimeError('Timestamp not found in Manifest')
if (self.max_age != 0 and

(datetime.datetime.utcnow() - ts.ts).days > self.max_age):
+   out.quiet = False
out.ewarn('Manifest is 
over %d days old, this is suspicious!' % (self.max_age,))
out.ewarn('You may want 
to try using another mirror and/or reporting this one:')
out.ewarn('  %s' % 
(dosyncuri,))
out.ewarn('')
+   out.quiet = quiet
 
out.einfo('Manifest timestamp: 
%s UTC' % (ts.ts,))
out.einfo('Valid OpenPGP 
signature found:')

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/

2018-05-02 Thread Zac Medico

commit: 10253819aae4cefee80f377e167ad521516d66a2
Author: Zac Medico  gentoo  org>
AuthorDate: Thu May  3 01:18:39 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu May  3 01:18:39 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=10253819

WriterPipeClosedTestCase: add_writer after pipe is filled

Hopefully this supresses a spurious writer callback observed
once for pypy in travis.

See: https://travis-ci.org/gentoo/portage/jobs/373734825

 pym/portage/tests/util/futures/asyncio/test_pipe_closed.py | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py 
b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
index 1ecddab78..5398ca35c 100644
--- a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
+++ b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
@@ -105,7 +105,6 @@ class WriterPipeClosedTestCase(_PipeClosedTestCase, 
TestCase):
 
writer_callback.called = loop.create_future()
_set_nonblocking(write_end.fileno())
-   loop.add_writer(write_end.fileno(), writer_callback)
 
# Fill up the pipe, so that no writer callbacks should 
be
# received until the state has changed.
@@ -117,6 +116,11 @@ class WriterPipeClosedTestCase(_PipeClosedTestCase, 
TestCase):
raise
break
 
+   # We've seen at least one spurious writer callback when
+   # this was registered before the pipe was filled, so
+   # register it afterwards.
+   loop.add_writer(write_end.fileno(), writer_callback)
+
# Allow the loop to check for IO events, and assert
# that our future is still not done.
loop.run_until_complete(asyncio.sleep(0, loop=loop))

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/emerge/, bin/

2018-05-01 Thread Zac Medico

commit: 4bda6c546aab816e835f62b326db9c2215e182ac
Author: Zac Medico  gentoo  org>
AuthorDate: Tue May  1 23:37:33 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue May  1 23:49:44 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4bda6c54

Revert "phase-helpers.sh: fix best/has_version -b for cross-prefix"

This reverts commit a0ac6e6727abec8d2482c95b1e84d8df24d78619,
since BROOT is only supposed to be set in src_* phases.
Update SimpleEmergeTestCase to call best/has_version -b only
in src_install for EAPI 7.

Reported-by: James Le Cuirot  gentoo.org>

 bin/phase-helpers.sh|  2 +-
 pym/portage/tests/emerge/test_simple.py | 23 ---
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh
index 8b16d7d31..59c19cf67 100644
--- a/bin/phase-helpers.sh
+++ b/bin/phase-helpers.sh
@@ -912,7 +912,7 @@ ___best_version_and_has_version_common() {
case ${root_arg} in
-r) root=${EROOT} ;;
-d) root=${ESYSROOT} ;;
-   -b) 
root=${BROOT:-/${PORTAGE_OVERRIDE_EPREFIX#/}} ;;
+   -b) root=${BROOT:-/} ;;
esac
else
case ${root_arg} in

diff --git a/pym/portage/tests/emerge/test_simple.py 
b/pym/portage/tests/emerge/test_simple.py
index 204c23296..b1402ddd5 100644
--- a/pym/portage/tests/emerge/test_simple.py
+++ b/pym/portage/tests/emerge/test_simple.py
@@ -58,6 +58,8 @@ src_install() {
echo "blah blah blah" > "${T}"/latin-1-$(printf "\\xa9")-regular-file 
|| die
doins "${T}"/latin-1-$(printf "\\xa9")-regular-file
dosym latin-1-$(printf "\\xa9")-regular-file 
${latin_1_dir}/latin-1-$(printf "\\xa9")-symlink || die
+
+   call_has_and_best_version
 }
 
 pkg_config() {
@@ -69,14 +71,29 @@ pkg_info() {
 }
 
 pkg_preinst() {
+   if ! ___eapi_best_version_and_has_version_support_-b_-d_-r; then
+   # The BROOT variable is unset during pkg_* phases for EAPI 7,
+   # therefore best/has_version -b is expected to fail if we 
attempt
+   # to call it for EAPI 7 here.
+   call_has_and_best_version
+   fi
+}
+
+call_has_and_best_version() {
local root_arg
if ___eapi_best_version_and_has_version_support_-b_-d_-r; then
root_arg="-b"
else
root_arg="--host-root"
fi
-   einfo "called pkg_preinst for $CATEGORY/$PF"
-
+   einfo "called ${EBUILD_PHASE_FUNC} for $CATEGORY/$PF"
+   einfo "EPREFIX=${EPREFIX}"
+   einfo "PORTAGE_OVERRIDE_EPREFIX=${PORTAGE_OVERRIDE_EPREFIX}"
+   einfo "ROOT=${ROOT}"
+   einfo "EROOT=${EROOT}"
+   einfo "SYSROOT=${SYSROOT}"
+   einfo "ESYSROOT=${ESYSROOT}"
+   einfo "BROOT=${BROOT}"
# Test that has_version and best_version work correctly with
# prefix (involves internal ROOT -> EROOT calculation in order
# to support ROOT override via the environment with EAPIs 3
@@ -90,7 +107,7 @@ pkg_preinst() {
if [[ ${EPREFIX} != ${PORTAGE_OVERRIDE_EPREFIX} ]] ; then
if has_version ${root_arg} $CATEGORY/$PN:$SLOT ; then
einfo "has_version ${root_arg} detects an installed 
instance of $CATEGORY/$PN:$SLOT"
-   einfo "best_version ${root_arg} reports that the 
installed instance is $(best_version $CATEGORY/$PN:$SLOT)"
+   einfo "best_version ${root_arg} reports that the 
installed instance is $(best_version ${root_arg} $CATEGORY/$PN:$SLOT)"
else
einfo "has_version ${root_arg} does not detect an 
installed instance of $CATEGORY/$PN:$SLOT"
fi

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

2018-05-01 Thread Zac Medico

commit: d8b9d5d6250a4f9ebaf8db6cbab96a585c56aadd
Author: Zac Medico  gentoo  org>
AuthorDate: Tue May  1 19:52:28 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue May  1 19:52:28 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8b9d5d6

rsync: surface key refresh exceptions early (bug 649276)

Since retry does not help for some types of errors, display errors as
soon as they occur.

Fixes: 5f29f03d0d97 ("rsync: add key refresh retry (bug 649276)")

 pym/portage/sync/modules/rsync/rsync.py | 14 +-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 180bf8571..b2850715e 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -157,9 +157,21 @@ class RsyncSync(NewBase):
if retry_decorator is None:
openpgp_env.refresh_keys()
else:
+   def noisy_refresh_keys():
+   """
+   Since retry does not 
help for some types of
+   errors, display errors 
as soon as they occur.
+   """
+   try:
+   
openpgp_env.refresh_keys()
+   except Exception as e:
+   
writemsg_level("%s\n" % (e,),
+   
level=logging.ERROR, noiselevel=-1)
+   raise # retry
+
loop = global_event_loop()
func_coroutine = 
functools.partial(loop.run_in_executor,
-   None, 
openpgp_env.refresh_keys)
+   None, 
noisy_refresh_keys)
decorated_func = 
retry_decorator(func_coroutine)

loop.run_until_complete(decorated_func())
out.eend(0)

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/emerge/, pym/portage/tests/resolver/, pym/portage/

2018-05-01 Thread Zac Medico

commit: 75d6df1a2988ba440feed3db02550b62ebe0c204
Author: Zac Medico  gentoo  org>
AuthorDate: Tue May  1 18:18:49 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue May  1 18:22:45 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=75d6df1a

Enable EAPI 7 (bug 654472)

Bug: https://bugs.gentoo.org/654472

 pym/portage/__init__.py | 2 +-
 pym/portage/const.py| 2 +-
 pym/portage/tests/emerge/test_simple.py | 6 +++---
 pym/portage/tests/resolver/test_eapi.py | 2 +-
 pym/portage/tests/resolver/test_required_use.py | 6 +++---
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 15c404f6f..166bfc700 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -462,7 +462,7 @@ def abssymlink(symlink, target=None):
 
 _doebuild_manifest_exempt_depend = 0
 
-_testing_eapis = frozenset(["4-python", "4-slot-abi", "5-progress", 
"5-hdepend", "7_pre1"])
+_testing_eapis = frozenset(["4-python", "4-slot-abi", "5-progress", 
"5-hdepend", "7_pre1", "7"])
 _deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1", "5_pre1", 
"5_pre2", "6_pre1"])
 _supported_eapis = frozenset([str(x) for x in range(portage.const.EAPI + 1)] + 
list(_testing_eapis) + list(_deprecated_eapis))
 

diff --git a/pym/portage/const.py b/pym/portage/const.py
index 16922a5e6..7f84bf0e9 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -204,7 +204,7 @@ SUPPORTED_FEATURES   = frozenset([
"xattr",
 ])
 
-EAPI = 6
+EAPI = 7
 
 HASHING_BLOCKSIZE= 32768
 

diff --git a/pym/portage/tests/emerge/test_simple.py 
b/pym/portage/tests/emerge/test_simple.py
index 495e22297..204c23296 100644
--- a/pym/portage/tests/emerge/test_simple.py
+++ b/pym/portage/tests/emerge/test_simple.py
@@ -1,4 +1,4 @@
-# Copyright 2011-2015 Gentoo Foundation
+# Copyright 2011-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import subprocess
@@ -116,13 +116,13 @@ pkg_preinst() {
"MISC_CONTENT": install_something,
},
"dev-libs/C-1": {
-   "EAPI" : "7_pre1",
+   "EAPI" : "7",
"KEYWORDS": "~x86",
"RDEPEND": "dev-libs/D[flag]",
"MISC_CONTENT": install_something,
},
"dev-libs/D-1": {
-   "EAPI" : "7_pre1",
+   "EAPI" : "7",
"KEYWORDS": "~x86",
"IUSE" : "flag",
"MISC_CONTENT": install_something,

diff --git a/pym/portage/tests/resolver/test_eapi.py 
b/pym/portage/tests/resolver/test_eapi.py
index fce05890b..50b9d90da 100644
--- a/pym/portage/tests/resolver/test_eapi.py
+++ b/pym/portage/tests/resolver/test_eapi.py
@@ -62,7 +62,7 @@ class EAPITestCase(TestCase):
 
#EAPI-7: implicit || ( ) no longer satisfies deps
"dev-libs/C-1": { "EAPI": "6", "IUSE": "foo", 
"RDEPEND": "|| ( foo? ( dev-libs/B ) )" }, 
-   "dev-libs/C-2": { "EAPI": "7_pre1", "IUSE": "foo", 
"RDEPEND": "|| ( foo? ( dev-libs/B ) )" }, 
+   "dev-libs/C-2": { "EAPI": "7", "IUSE": "foo", 
"RDEPEND": "|| ( foo? ( dev-libs/B ) )" },
}
 
test_cases = (

diff --git a/pym/portage/tests/resolver/test_required_use.py 
b/pym/portage/tests/resolver/test_required_use.py
index 7909f927f..c679ce300 100644
--- a/pym/portage/tests/resolver/test_required_use.py
+++ b/pym/portage/tests/resolver/test_required_use.py
@@ -51,9 +51,9 @@ class RequiredUSETestCase(TestCase):
"dev-libs/E-4" : {"EAPI": "5", "IUSE": "+foo +bar", 
"REQUIRED_USE": "?? ( foo bar )"},
"dev-libs/E-5" : {"EAPI": "5", "IUSE": "+foo +bar", 
"REQUIRED_USE": "?? ( )"},
 
-   "dev-libs/F-1" : {"EAPI": "7_pre1", "IUSE": "+foo 
+bar", "REQUIRED_USE": "|| ( )"},
-   "dev-libs/F-2" : {"EAPI": "7_pre1", "IUSE": "+foo 
+bar", "REQUIRED_USE": "^^ ( )"},
-   "dev-libs/F-3" : {"EAPI": "7_pre1", "IUSE": "+foo 
+bar", "REQUIRED_USE": "?? ( )"},
+   "dev-libs/F-1" : {"EAPI": "7", "IUSE": "+foo +bar", 
"REQUIRED_USE": "|| ( )"},
+   "dev-libs/F-2" : {"EAPI": "7", "IUSE": "+foo +bar", 
"REQUIRED_USE": "^^ ( )"},
+   "dev-libs/F-3" : {"EAPI": "7", "IUSE": "+foo +bar", 
"REQUIRED_USE": "?? ( )"},
}
 
test_cases = (

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/emerge/, bin/

2018-05-01 Thread Zac Medico

commit: a0ac6e6727abec8d2482c95b1e84d8df24d78619
Author: Zac Medico  gentoo  org>
AuthorDate: Tue May  1 18:14:07 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue May  1 18:15:47 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a0ac6e67

phase-helpers.sh: fix best/has_version -b for cross-prefix

Fixes: 43b6be7423aa ("phase-helpers.sh: Implement -r|-d|-b options for 
best/has_version")

 bin/phase-helpers.sh|  2 +-
 pym/portage/tests/emerge/test_simple.py | 12 
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh
index 59c19cf67..8b16d7d31 100644
--- a/bin/phase-helpers.sh
+++ b/bin/phase-helpers.sh
@@ -912,7 +912,7 @@ ___best_version_and_has_version_common() {
case ${root_arg} in
-r) root=${EROOT} ;;
-d) root=${ESYSROOT} ;;
-   -b) root=${BROOT:-/} ;;
+   -b) 
root=${BROOT:-/${PORTAGE_OVERRIDE_EPREFIX#/}} ;;
esac
else
case ${root_arg} in

diff --git a/pym/portage/tests/emerge/test_simple.py 
b/pym/portage/tests/emerge/test_simple.py
index 17dcd548d..495e22297 100644
--- a/pym/portage/tests/emerge/test_simple.py
+++ b/pym/portage/tests/emerge/test_simple.py
@@ -119,11 +119,13 @@ pkg_preinst() {
"EAPI" : "7_pre1",
"KEYWORDS": "~x86",
"RDEPEND": "dev-libs/D[flag]",
+   "MISC_CONTENT": install_something,
},
"dev-libs/D-1": {
"EAPI" : "7_pre1",
"KEYWORDS": "~x86",
"IUSE" : "flag",
+   "MISC_CONTENT": install_something,
},
"virtual/foo-0": {
"EAPI" : "5",
@@ -326,6 +328,16 @@ pkg_preinst() {
portageq_cmd + ("match", eroot, "dev-libs/D[flag]"),
 
# Test cross-prefix usage, including chpathtool for 
binpkgs.
+   # EAPI 7
+   ({"EPREFIX" : cross_prefix},) + \
+   emerge_cmd + ("dev-libs/C",),
+   ({"EPREFIX" : cross_prefix},) + \
+   portageq_cmd + ("has_version", cross_prefix, 
"dev-libs/C"),
+   ({"EPREFIX" : cross_prefix},) + \
+   portageq_cmd + ("has_version", cross_prefix, 
"dev-libs/D"),
+   ({"ROOT": cross_root},) + emerge_cmd + ("dev-libs/D",),
+   portageq_cmd + ("has_version", cross_eroot, 
"dev-libs/D"),
+   # EAPI 5
({"EPREFIX" : cross_prefix},) + \
emerge_cmd + ("--usepkgonly", "dev-libs/A"),
({"EPREFIX" : cross_prefix},) + \

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/emerge/

2018-05-01 Thread Zac Medico

commit: 844998daa50e9db0a337bc0cf9f665d224779665
Author: Zac Medico  gentoo  org>
AuthorDate: Tue May  1 17:49:07 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue May  1 17:52:02 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=844998da

SimpleEmergeTestCase: EAPI 7_pre1 has/best_version -b

 pym/portage/tests/emerge/test_simple.py | 18 --
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/pym/portage/tests/emerge/test_simple.py 
b/pym/portage/tests/emerge/test_simple.py
index f99c77927..17dcd548d 100644
--- a/pym/portage/tests/emerge/test_simple.py
+++ b/pym/portage/tests/emerge/test_simple.py
@@ -69,6 +69,12 @@ pkg_info() {
 }
 
 pkg_preinst() {
+   local root_arg
+   if ___eapi_best_version_and_has_version_support_-b_-d_-r; then
+   root_arg="-b"
+   else
+   root_arg="--host-root"
+   fi
einfo "called pkg_preinst for $CATEGORY/$PF"
 
# Test that has_version and best_version work correctly with
@@ -82,11 +88,11 @@ pkg_preinst() {
einfo "has_version does not detect an installed instance of 
$CATEGORY/$PN:$SLOT"
fi
if [[ ${EPREFIX} != ${PORTAGE_OVERRIDE_EPREFIX} ]] ; then
-   if has_version --host-root $CATEGORY/$PN:$SLOT ; then
-   einfo "has_version --host-root detects an installed 
instance of $CATEGORY/$PN:$SLOT"
-   einfo "best_version --host-root reports that the 
installed instance is $(best_version $CATEGORY/$PN:$SLOT)"
+   if has_version ${root_arg} $CATEGORY/$PN:$SLOT ; then
+   einfo "has_version ${root_arg} detects an installed 
instance of $CATEGORY/$PN:$SLOT"
+   einfo "best_version ${root_arg} reports that the 
installed instance is $(best_version $CATEGORY/$PN:$SLOT)"
else
-   einfo "has_version --host-root does not detect an 
installed instance of $CATEGORY/$PN:$SLOT"
+   einfo "has_version ${root_arg} does not detect an 
installed instance of $CATEGORY/$PN:$SLOT"
fi
fi
 }
@@ -110,12 +116,12 @@ pkg_preinst() {
"MISC_CONTENT": install_something,
},
"dev-libs/C-1": {
-   "EAPI" : "6",
+   "EAPI" : "7_pre1",
"KEYWORDS": "~x86",
"RDEPEND": "dev-libs/D[flag]",
},
"dev-libs/D-1": {
-   "EAPI" : "6",
+   "EAPI" : "7_pre1",
"KEYWORDS": "~x86",
"IUSE" : "flag",
},

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/_ipc/

2018-05-01 Thread Zac Medico

commit: 107dca9a3171e4870038769aef66612209ecd423
Author: Zac Medico  gentoo  org>
AuthorDate: Tue May  1 17:27:06 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue May  1 17:27:06 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=107dca9a

QueryCommand: handle empty *ROOT values for EAPI 7

The absence of a trailing slash in EAPI 7 means that it's
valid for *ROOT values to be empty, so translate the empty
value to a single slash where appropriate. This solves
errors like the following:

best_version: Invalid ROOT: ./

See: 6cb70f9ef0e7 ("Strip trailing slash from D, ED, ROOT, EROOT in EAPI 7")

 pym/portage/package/ebuild/_ipc/QueryCommand.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/package/ebuild/_ipc/QueryCommand.py 
b/pym/portage/package/ebuild/_ipc/QueryCommand.py
index 351c95628..fa6d1ea16 100644
--- a/pym/portage/package/ebuild/_ipc/QueryCommand.py
+++ b/pym/portage/package/ebuild/_ipc/QueryCommand.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2013 Gentoo Foundation
+# Copyright 2010-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -49,7 +49,7 @@ class QueryCommand(IpcCommand):
db = self.get_db()
eapi = self.settings.get('EAPI')
 
-   root = normalize_path(root).rstrip(os.path.sep) + os.path.sep
+   root = normalize_path(root or os.sep).rstrip(os.sep) + os.sep
if root not in db:
return ('', '%s: Invalid ROOT: %s\n' % (cmd, root), 3)

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/_config/, repoman/man/, pym/portage/package/ebuild/, ...

2018-05-01 Thread Zac Medico

commit: d8d47bf451bb3fd6bce1cdb035a5f12253f5a167
Author: James Le Cuirot  gentoo  org>
AuthorDate: Sat Sep 30 20:45:54 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue May  1 08:28:43 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8d47bf4

Export BROOT to ebuild env in EAPI 7

Export the BROOT variable corresponding to the location where BDEPEND
are installed.

Bug: https://bugs.gentoo.org/317337

 bin/eapi.sh|  4 
 bin/ebuild.sh  |  3 +++
 bin/phase-functions.sh |  3 +++
 man/ebuild.5   |  6 ++
 pym/portage/eapi.py|  7 ++-
 pym/portage/package/ebuild/_config/LocationsManager.py |  4 
 pym/portage/package/ebuild/_config/special_env_vars.py |  4 ++--
 pym/portage/package/ebuild/config.py   | 10 +-
 repoman/cnf/qa_data/qa_data.yaml   |  2 +-
 repoman/man/repoman.1  |  2 +-
 .../pym/repoman/modules/linechecks/assignment/assignment.py|  9 -
 repoman/pym/repoman/modules/linechecks/quotes/quotes.py|  3 ++-
 12 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/bin/eapi.sh b/bin/eapi.sh
index b4d8c9a90..3f4c9691b 100644
--- a/bin/eapi.sh
+++ b/bin/eapi.sh
@@ -30,6 +30,10 @@ ___eapi_has_prefix_variables() {
[[ ! ${1-${EAPI-0}} =~ ^(0|1|2)$ || " ${FEATURES} " == *" force-prefix 
"* ]]
 }
 
+___eapi_has_BROOT() {
+   [[ ! ${1-${EAPI-0}} =~ 
^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend|5-progress|6)$ ]]
+}
+
 ___eapi_has_SYSROOT() {
[[ ! ${1-${EAPI-0}} =~ 
^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend|5-progress|6)$ ]]
 }

diff --git a/bin/ebuild.sh b/bin/ebuild.sh
index 11441bfb2..98ed570c2 100755
--- a/bin/ebuild.sh
+++ b/bin/ebuild.sh
@@ -769,6 +769,9 @@ else
if ___eapi_has_prefix_variables; then
declare -r ED EPREFIX EROOT
fi
+   if ___eapi_has_BROOT; then
+   declare -r BROOT
+   fi
 
# If ${EBUILD_FORCE_TEST} == 1 and USE came from ${T}/environment
# then it might not have USE=test like it's supposed to here.

diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index 1e096cddc..1f9faaa41 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -100,6 +100,9 @@ __filter_readonly_variables() {
filtered_vars="$readonly_bash_vars $bash_misc_vars
$PORTAGE_READONLY_VARS $misc_garbage_vars"
 
+   if ___eapi_has_BROOT; then
+   filtered_vars+=" BROOT"
+   fi
if ___eapi_has_SYSROOT; then
filtered_vars+=" SYSROOT"
fi

diff --git a/man/ebuild.5 b/man/ebuild.5
index eb27d43bb..9f491dd73 100644
--- a/man/ebuild.5
+++ b/man/ebuild.5
@@ -527,6 +527,12 @@ Beginning with \fBEAPI 3\fR, contains
 "${ROOT%/}${EPREFIX}/" for convenience
 purposes. Do not modify this variable.
 .TP
+.B BROOT\fR = \fI"${EPREFIX}"
+Beginning with \fBEAPI 7\fR, the absolute path to the root directory
+containing build dependencies satisfied by BDEPEND, typically
+executable build tools. This includes any applicable offset prefix. Do
+not modify this variable.
+.TP
 .B DESCRIPTION\fR = \fI"A happy little package"
 Should contain a short description of the package.
 .TP

diff --git a/pym/portage/eapi.py b/pym/portage/eapi.py
index f34e19ac9..158d58243 100644
--- a/pym/portage/eapi.py
+++ b/pym/portage/eapi.py
@@ -123,13 +123,17 @@ def eapi_path_variables_end_with_trailing_slash(eapi):
return eapi in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi",
"5", "5-progress", "6")
 
+def eapi_has_broot(eapi):
+   return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi",
+   "5", "5-progress", "5-hdepend", "6")
+
 def eapi_has_sysroot(eapi):
return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi",
"5", "5-progress", "5-hdepend", "6")
 
 _eapi_attrs = collections.namedtuple('_eapi_attrs',
'allows_package_provided '
-   'bdepend dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC '
+   'bdepend broot dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC '
'exports_PORTDIR exports_ECLASSDIR '
'feature_flag_test feature_flag_targetroot '
'hdepend iuse_defaults iuse_effective posixish_locale '
@@ -159,6 +163,7 @@ def _get_eapi_attrs(eapi):
eapi_attrs = _eapi_attrs(
allows_package_provided=(eapi is None or 
eapi_allows_package_provided(eapi)),
bdepend = (eapi is not None and eapi_has_bdepend(eapi)),
+   broot = (eapi is None or eapi_has_broot(eapi)),
dots_in_PN = (eapi is None or eapi_allows_dots_in_PN(eapi)),

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/_config/, pym/_emerge/, pym/portage/, bin/, ...

2018-05-01 Thread Zac Medico

commit: a41dacf7926c0242678300a8e1770df84029e8a4
Author: James Le Cuirot  gentoo  org>
AuthorDate: Mon Apr 23 21:49:43 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue May  1 08:28:43 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a41dacf7

Export SYSROOT and ESYSROOT in ebuild env in EAPI 7

SYSROOT is the new destination for DEPEND, however it must currently
be equal either / or ROOT. This is partly to simplify the
implementation and partly because supporting a third separate
destination would not be of much practical benefit anyway.

Bug: https://bugs.gentoo.org/317337

 bin/eapi.sh|  4 
 bin/phase-functions.sh |  6 ++
 pym/_emerge/actions.py |  2 +-
 pym/_emerge/depgraph.py|  4 +++-
 pym/portage/__init__.py|  6 +++---
 pym/portage/_legacy_globals.py |  3 ++-
 pym/portage/eapi.py|  7 ++-
 .../package/ebuild/_config/LocationsManager.py | 19 +++--
 pym/portage/package/ebuild/config.py   | 24 +++---
 9 files changed, 63 insertions(+), 12 deletions(-)

diff --git a/bin/eapi.sh b/bin/eapi.sh
index 40af7b776..b4d8c9a90 100644
--- a/bin/eapi.sh
+++ b/bin/eapi.sh
@@ -30,6 +30,10 @@ ___eapi_has_prefix_variables() {
[[ ! ${1-${EAPI-0}} =~ ^(0|1|2)$ || " ${FEATURES} " == *" force-prefix 
"* ]]
 }
 
+___eapi_has_SYSROOT() {
+   [[ ! ${1-${EAPI-0}} =~ 
^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend|5-progress|6)$ ]]
+}
+
 ___eapi_has_HDEPEND() {
[[ ${1-${EAPI-0}} =~ ^(5-hdepend)$ ]]
 }

diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index 3de8d01b5..1e096cddc 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -100,10 +100,16 @@ __filter_readonly_variables() {
filtered_vars="$readonly_bash_vars $bash_misc_vars
$PORTAGE_READONLY_VARS $misc_garbage_vars"
 
+   if ___eapi_has_SYSROOT; then
+   filtered_vars+=" SYSROOT"
+   fi
# Don't filter/interfere with prefix variables unless they are
# supported by the current EAPI.
if ___eapi_has_prefix_variables; then
filtered_vars+=" ED EPREFIX EROOT"
+   if ___eapi_has_SYSROOT; then
+   filtered_vars+=" ESYSROOT"
+   fi
fi
if ___eapi_has_PORTDIR_ECLASSDIR; then
filtered_vars+=" PORTDIR ECLASSDIR"

diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 1a289f523..70fb8d3b4 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -2438,7 +2438,7 @@ def load_emerge_config(emerge_config=None, env=None, 
**kargs):
env = os.environ if env is None else env
kwargs = {'env': env}
for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), 
("target_root", "ROOT"),
-   ("eprefix", "EPREFIX")):
+   ("sysroot", "SYSROOT"), ("eprefix", "EPREFIX")):
v = env.get(envvar)
if v and v.strip():
kwargs[k] = v

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 67f912f5e..fbd16ad98 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -3322,8 +3322,10 @@ class depgraph(object):
if removal_action:
depend_root = myroot
else:
-   if eapi_attrs.bdepend or eapi_attrs.hdepend:
+   if eapi_attrs.hdepend:
depend_root = myroot
+   elif eapi_attrs.bdepend:
+   depend_root = 
pkg.root_config.settings["ESYSROOT"]
else:
depend_root = 
self._frozen_config._running_root.root
root_deps = 
self._frozen_config.myopts.get("--root-deps")

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 137351a69..15c404f6f 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -525,7 +525,7 @@ class _trees_dict(dict):
self._target_eroot = None
 
 def create_trees(config_root=None, target_root=None, trees=None, env=None,
-   eprefix=None):
+   sysroot=None, eprefix=None):
 
if trees is None:
trees = _trees_dict()
@@ -538,7 +538,7 @@ def create_trees(config_root=None, target_root=None, 
trees=None, env=None,
env = os.environ
 
settings = config(config_root=config_root, target_root=target_root,
-   env=env, eprefix=eprefix)
+   env=env, sysroot=sysroot, eprefix=eprefix)
settings.lock()
 
depcachedir = settings.get('PORTAGE_DEPCACHEDIR')
@@ -562,7 +562,7 @@ def create_trees(config_root=None, target_root=None, 
trees=None, env=None,
if

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/, pym/portage/dbapi/, pym/portage/util/futures/, ...

2018-05-01 Thread Zac Medico

commit: 90d78484d6be481a9caf22c017c62ea43f8ffe33
Author: Zac Medico  gentoo  org>
AuthorDate: Tue May  1 06:22:10 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue May  1 06:46:27 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=90d78484

_PortageEventLoop: add _asyncio_* properties for internal use

It's better to avoid accessing the _PortageEventLoop._loop
attribute which exposes *all* EventLoop methods, therefore expose
_asyncio_child_watcher and _asyncio_wrapper attributes for use by
portage internals, providing minimal compatibility between
_PortageEventloop and EventLoop.

 pym/portage/dbapi/porttree.py  |  2 +-
 .../util/futures/asyncio/test_subprocess_exec.py   |  2 +-
 .../tests/util/futures/test_iter_completed.py  |  2 +-
 pym/portage/util/futures/executor/fork.py  |  2 +-
 pym/portage/util/futures/iter_completed.py |  2 +-
 pym/portage/util/futures/unix_events.py| 22 ++
 6 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 3ce214cd7..6c38232bb 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -667,7 +667,7 @@ class portdbapi(dbapi):
 
proc = EbuildMetadataPhase(cpv=mycpv,
ebuild_hash=ebuild_hash, portdb=self,
-   repo_path=mylocation, scheduler=loop._loop,
+   repo_path=mylocation, scheduler=loop,
settings=self.doebuild_settings)
 
proc.addExitListener(functools.partial(self._aux_get_return,

diff --git a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py 
b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
index 8c8c395ca..be103a9e0 100644
--- a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
+++ b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
@@ -34,7 +34,7 @@ class _Reader(object):
def __init__(self, future, input_file, loop):
self._future = future
self._pipe_reader = PipeReader(
-   input_files={'input_file':input_file}, 
scheduler=loop._loop)
+   input_files={'input_file':input_file}, scheduler=loop)
 
self._future.add_done_callback(self._cancel_callback)
self._pipe_reader.addExitListener(self._eof)

diff --git a/pym/portage/tests/util/futures/test_iter_completed.py 
b/pym/portage/tests/util/futures/test_iter_completed.py
index 71343c22d..90668eb02 100644
--- a/pym/portage/tests/util/futures/test_iter_completed.py
+++ b/pym/portage/tests/util/futures/test_iter_completed.py
@@ -46,7 +46,7 @@ class IterCompletedTestCase(TestCase):
def future_generator():
for task in tasks:
task.future = loop.create_future()
-   task.scheduler = loop._loop
+   task.scheduler = loop
task.start()
yield task.future
 

diff --git a/pym/portage/util/futures/executor/fork.py 
b/pym/portage/util/futures/executor/fork.py
index 51367f934..81c292e2c 100644
--- a/pym/portage/util/futures/executor/fork.py
+++ b/pym/portage/util/futures/executor/fork.py
@@ -54,7 +54,7 @@ class ForkExecutor(object):
future, proc = self._submit_queue.popleft()

future.add_done_callback(functools.partial(self._cancel_cb, proc))
proc.addExitListener(functools.partial(self._proc_exit, 
future))
-   proc.scheduler = self._loop._loop
+   proc.scheduler = self._loop
proc.start()
self._running_tasks[id(proc)] = proc
 

diff --git a/pym/portage/util/futures/iter_completed.py 
b/pym/portage/util/futures/iter_completed.py
index 1d6a9a4bd..8b0f417d9 100644
--- a/pym/portage/util/futures/iter_completed.py
+++ b/pym/portage/util/futures/iter_completed.py
@@ -77,7 +77,7 @@ def async_iter_completed(futures, max_jobs=None, 
max_load=None, loop=None):
task_generator(),
max_jobs=max_jobs,
max_load=max_load,
-   event_loop=loop._loop)
+   event_loop=loop)
 
def done_callback(future_done_set, wait_result):
"""Propagate results from wait_result to future_done_set."""

diff --git a/pym/portage/util/futures/unix_events.py 
b/pym/portage/util/futures/unix_events.py
index 1a86ed439..00f522b61 100644
--- a/pym/portage/util/futures/unix_events.py
+++ b/pym/portage/util/futures/unix_events.py
@@ -72,6 +72,28 @@ class _PortageEventLoop(events.AbstractEventLoop):
self.set_debug = loop.set_debug
self.get_debug = loop.get_debug
 
+   @property
+   def _asyncio_child_watcher(self):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/, pym/portage/util/_async/

2018-04-30 Thread Zac Medico

commit: 8b3dd3537a2c50035d6369946e65305d448d7f1b
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Apr 30 19:11:52 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Apr 30 19:25:20 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8b3dd353

AbstractPollTask: remove deprecated _read_* event parameter

Fixes: c11a6ec05f02 ("AbstractPollTask: add_reader asyncio compat (bug 654382)")

 pym/_emerge/AbstractPollTask.py   | 6 ++
 pym/_emerge/AsynchronousLock.py   | 2 +-
 pym/_emerge/EbuildIpcDaemon.py| 2 +-
 pym/_emerge/EbuildMetadataPhase.py| 2 +-
 pym/_emerge/PipeReader.py | 4 ++--
 pym/portage/dbapi/_MergeProcess.py| 2 +-
 pym/portage/util/_async/PipeLogger.py | 2 +-
 7 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py
index f68cd3a15..6cccb27d9 100644
--- a/pym/_emerge/AbstractPollTask.py
+++ b/pym/_emerge/AbstractPollTask.py
@@ -19,7 +19,7 @@ class AbstractPollTask(AsynchronousTask):
def isAlive(self):
return bool(self._registered)
 
-   def _read_array(self, f, event):
+   def _read_array(self, f):
"""
NOTE: array.fromfile() is used here only for testing purposes,
because it has bugs in all known versions of Python (including
@@ -65,7 +65,7 @@ class AbstractPollTask(AsynchronousTask):
 
return buf
 
-   def _read_buf(self, fd, event):
+   def _read_buf(self, fd):
"""
Read self._bufsize into a string of bytes, handling EAGAIN and
EIO. This will only call os.read() once, so the caller should
@@ -78,8 +78,6 @@ class AbstractPollTask(AsynchronousTask):
 
@param fd: file descriptor (non-blocking mode required)
@type fd: int
-   @param event: poll event flags
-   @type event: int
@rtype: bytes or None
@return: A string of bytes, or None
"""

diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py
index 6cf37369f..aed1bcb15 100644
--- a/pym/_emerge/AsynchronousLock.py
+++ b/pym/_emerge/AsynchronousLock.py
@@ -266,7 +266,7 @@ class _LockProcess(AbstractPollTask):
return self.returncode
 
def _output_handler(self):
-   buf = self._read_buf(self._files['pipe_in'], None)
+   buf = self._read_buf(self._files['pipe_in'])
if buf:
self._acquired = True
self._unregister()

diff --git a/pym/_emerge/EbuildIpcDaemon.py b/pym/_emerge/EbuildIpcDaemon.py
index c16049ee4..d0dbe18bd 100644
--- a/pym/_emerge/EbuildIpcDaemon.py
+++ b/pym/_emerge/EbuildIpcDaemon.py
@@ -34,7 +34,7 @@ class EbuildIpcDaemon(FifoIpcDaemon):
 
def _input_handler(self):
# Read the whole pickle in a single atomic read() call.
-   data = self._read_buf(self._files.pipe_in, None)
+   data = self._read_buf(self._files.pipe_in)
if data is None:
pass # EAGAIN
elif data:

diff --git a/pym/_emerge/EbuildMetadataPhase.py 
b/pym/_emerge/EbuildMetadataPhase.py
index 42bcd6739..4940d40b6 100644
--- a/pym/_emerge/EbuildMetadataPhase.py
+++ b/pym/_emerge/EbuildMetadataPhase.py
@@ -130,7 +130,7 @@ class EbuildMetadataPhase(SubProcess):
 
def _output_handler(self):
while True:
-   buf = self._read_buf(self._files.ebuild, None)
+   buf = self._read_buf(self._files.ebuild)
if buf is None:
break # EAGAIN
elif buf:

diff --git a/pym/_emerge/PipeReader.py b/pym/_emerge/PipeReader.py
index 6a3fc9ea6..151be94ea 100644
--- a/pym/_emerge/PipeReader.py
+++ b/pym/_emerge/PipeReader.py
@@ -61,7 +61,7 @@ class PipeReader(AbstractPollTask):
def _output_handler(self, fd):
 
while True:
-   data = self._read_buf(fd, None)
+   data = self._read_buf(fd)
if data is None:
break
if data:
@@ -79,7 +79,7 @@ class PipeReader(AbstractPollTask):
break
 
while True:
-   data = self._read_array(f, None)
+   data = self._read_array(f)
if data is None:
break
if data:

diff --git a/pym/portage/dbapi/_MergeProcess.py 
b/pym/portage/dbapi/_MergeProcess.py
index 42f2d84e5..fefdf8635 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -79,7 +79,7 @@ class MergeProcess(ForkProcess):
self._locked_vdb = False
 
def _elog_output_handler(self):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

2018-04-30 Thread Zac Medico

commit: 41af82685d688cb03da743cdd03295271a3ef09c
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Apr 30 05:45:13 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Apr 30 06:20:01 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=41af8268

_MergeProcess: add_reader asyncio compat (bug 654382)

Use add_reader for asyncio compatibility.

Bug: https://bugs.gentoo.org/654382

 pym/portage/dbapi/_MergeProcess.py | 27 +++
 1 file changed, 7 insertions(+), 20 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py 
b/pym/portage/dbapi/_MergeProcess.py
index bfbe387e4..42f2d84e5 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -7,7 +7,6 @@ import signal
 import sys
 import traceback
 
-import errno
 import fcntl
 import portage
 from portage import os, _unicode_decode
@@ -24,7 +23,7 @@ class MergeProcess(ForkProcess):
__slots__ = ('mycat', 'mypkg', 'settings', 'treetype',
'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild',
'mydbapi', 'postinst_failure', 'prev_mtimes', 'unmerge',
-   '_elog_reader_fd', '_elog_reg_id',
+   '_elog_reader_fd',
'_buf', '_elog_keys', '_locked_vdb')
 
def _start(self):
@@ -79,14 +78,8 @@ class MergeProcess(ForkProcess):
self.vartree.dbapi.unlock()
self._locked_vdb = False
 
-   def _elog_output_handler(self, fd, event):
-   output = None
-   if event & self.scheduler.IO_IN:
-   try:
-   output = os.read(fd, self._bufsize)
-   except OSError as e:
-   if e.errno not in (errno.EAGAIN, errno.EINTR):
-   raise
+   def _elog_output_handler(self):
+   output = self._read_buf(self._elog_reader_fd, None)
if output:
lines = _unicode_decode(output).split('\n')
if len(lines) == 1:
@@ -101,15 +94,12 @@ class MergeProcess(ForkProcess):
reporter = 
getattr(portage.elog.messages, funcname)
reporter(msg, phase=phase, key=key, 
out=out)
 
-   if event & self.scheduler.IO_HUP:
-   self.scheduler.source_remove(self._elog_reg_id)
-   self._elog_reg_id = None
+   elif output is not None: # EIO/POLLHUP
+   self.scheduler.remove_reader(self._elog_reader_fd)
os.close(self._elog_reader_fd)
self._elog_reader_fd = None
return False
 
-   return True
-
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
@@ -142,8 +132,7 @@ class MergeProcess(ForkProcess):
treetype=self.treetype, vartree=self.vartree,
blockers=blockers, pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
-   self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd,
-   self._registered_events, self._elog_output_handler)
+   self.scheduler.add_reader(elog_reader_fd, 
self._elog_output_handler)
 
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
@@ -275,10 +264,8 @@ class MergeProcess(ForkProcess):
pass
 
self._unlock_vdb()
-   if self._elog_reg_id is not None:
-   self.scheduler.source_remove(self._elog_reg_id)
-   self._elog_reg_id = None
if self._elog_reader_fd is not None:
+   self.scheduler.remove_reader(self._elog_reader_fd)
os.close(self._elog_reader_fd)
self._elog_reader_fd = None
if self._elog_keys is not None:

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-30 Thread Zac Medico

commit: 2495fe6ff060e2ed8ee54e08a4dec132de1f4984
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Apr 30 05:52:07 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Apr 30 06:20:01 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2495fe6f

PipeLogger: add_reader asyncio compat (bug 654382)

Use add_reader for asyncio compatibility.

Bug: https://bugs.gentoo.org/654382

 pym/portage/util/_async/PipeLogger.py | 20 ++--
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/pym/portage/util/_async/PipeLogger.py 
b/pym/portage/util/_async/PipeLogger.py
index 02de74b16..61000271d 100644
--- a/pym/portage/util/_async/PipeLogger.py
+++ b/pym/portage/util/_async/PipeLogger.py
@@ -21,7 +21,7 @@ class PipeLogger(AbstractPollTask):
"""
 
__slots__ = ("input_fd", "log_file_path", "stdout_fd") + \
-   ("_log_file", "_log_file_real", "_reg_id")
+   ("_log_file", "_log_file_real")
 
def _start(self):
 
@@ -57,8 +57,7 @@ class PipeLogger(AbstractPollTask):
fcntl.fcntl(fd, fcntl.F_SETFD,
fcntl.fcntl(fd, fcntl.F_GETFD) | 
fcntl.FD_CLOEXEC)
 
-   self._reg_id = self.scheduler.io_add_watch(fd,
-   self._registered_events, self._output_handler)
+   self.scheduler.add_reader(fd, self._output_handler, fd)
self._registered = True
 
def _cancel(self):
@@ -66,14 +65,14 @@ class PipeLogger(AbstractPollTask):
if self.returncode is None:
self.returncode = self._cancelled_returncode
 
-   def _output_handler(self, fd, event):
+   def _output_handler(self, fd):
 
background = self.background
stdout_fd = self.stdout_fd
log_file = self._log_file 
 
while True:
-   buf = self._read_buf(fd, event)
+   buf = self._read_buf(fd, None)
 
if buf is None:
# not a POLLIN event, EAGAIN, etc...
@@ -124,20 +123,13 @@ class PipeLogger(AbstractPollTask):
log_file.write(buf)
log_file.flush()
 
-   self._unregister_if_appropriate(event)
-
-   return True
-
def _unregister(self):
-
-   if self._reg_id is not None:
-   self.scheduler.source_remove(self._reg_id)
-   self._reg_id = None
-
if self.input_fd is not None:
if isinstance(self.input_fd, int):
+   self.scheduler.remove_reader(self.input_fd)
os.close(self.input_fd)
else:
+   
self.scheduler.remove_reader(self.input_fd.fileno())
self.input_fd.close()
self.input_fd = None

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/, pym/portage/util/_eventloop/

2018-04-29 Thread Zac Medico

commit: c77afbc31fa687cc612a6f946b324bf4d74d8175
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Apr 30 01:49:18 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Apr 30 02:14:41 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c77afbc3

EventLoop: call add_reader/writer callbacks after pipe is closed (bug 654382)

Callbacks registered via add_reader/writer methods need to be called
when the other end of a pipe is closed, which does not result in a
normal read or write event. Therefore, respond to other event types
as well, for compatibility with the asyncio event loop implementation.

The included unit tests demonstrate asyncio compatible behavior for
both reader and writer callbacks.

Bug: https://bugs.gentoo.org/654382

 .../tests/util/futures/asyncio/test_pipe_closed.py | 133 +
 pym/portage/util/_eventloop/EventLoop.py   |   7 +-
 2 files changed, 138 insertions(+), 2 deletions(-)

diff --git a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py 
b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
new file mode 100644
index 0..1ecddab78
--- /dev/null
+++ b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
@@ -0,0 +1,133 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import os
+import pty
+import shutil
+import socket
+import sys
+import tempfile
+
+from portage.tests import TestCase
+from portage.util.futures import asyncio
+from portage.util.futures.unix_events import (
+   DefaultEventLoopPolicy,
+   _set_nonblocking,
+)
+
+
+class _PipeClosedTestCase(object):
+
+   def test_pipe(self):
+   read_end, write_end = os.pipe()
+   self._do_test(read_end, write_end)
+
+   def test_pty_device(self):
+   try:
+   read_end, write_end = pty.openpty()
+   except EnvironmentError:
+   self.skipTest('pty not available')
+   self._do_test(read_end, write_end)
+
+   def test_domain_socket(self):
+   if sys.version_info >= (3, 2):
+   read_end, write_end = socket.socketpair()
+   else:
+   self.skipTest('socket detach not supported')
+   self._do_test(read_end.detach(), write_end.detach())
+
+   def test_named_pipe(self):
+   tempdir = tempfile.mkdtemp()
+   try:
+   fifo_path = os.path.join(tempdir, 'fifo')
+   os.mkfifo(fifo_path)
+   self._do_test(os.open(fifo_path, 
os.O_NONBLOCK|os.O_RDONLY),
+   os.open(fifo_path, os.O_NONBLOCK|os.O_WRONLY))
+   finally:
+   shutil.rmtree(tempdir)
+
+
+class ReaderPipeClosedTestCase(_PipeClosedTestCase, TestCase):
+   """
+   Test that a reader callback is called after the other end of
+   the pipe has been closed.
+   """
+   def _do_test(self, read_end, write_end):
+   initial_policy = asyncio.get_event_loop_policy()
+   if not isinstance(initial_policy, DefaultEventLoopPolicy):
+   asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
+
+   loop = asyncio.get_event_loop()
+   read_end = os.fdopen(read_end, 'rb', 0)
+   write_end = os.fdopen(write_end, 'wb', 0)
+   try:
+   def reader_callback():
+   if not reader_callback.called.done():
+   reader_callback.called.set_result(None)
+
+   reader_callback.called = loop.create_future()
+   loop.add_reader(read_end.fileno(), reader_callback)
+
+   # Allow the loop to check for IO events, and assert
+   # that our future is still not done.
+   loop.run_until_complete(asyncio.sleep(0, loop=loop))
+   self.assertFalse(reader_callback.called.done())
+
+   # Demonstrate that the callback is called afer the
+   # other end of the pipe has been closed.
+   write_end.close()
+   loop.run_until_complete(reader_callback.called)
+   finally:
+   loop.remove_reader(read_end.fileno())
+   write_end.close()
+   read_end.close()
+   asyncio.set_event_loop_policy(initial_policy)
+
+
+class WriterPipeClosedTestCase(_PipeClosedTestCase, TestCase):
+   """
+   Test that a writer callback is called after the other end of
+   the pipe has been closed.
+   """
+   def _do_test(self, read_end, write_end):
+   initial_policy = asyncio.get_event_loop_policy()
+   if not isinstance(initial_policy,

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-29 Thread Zac Medico

commit: abf4b7abcdc53bbbd86eb6f79d45417b54ed64d2
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 29 22:12:13 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Apr 29 22:18:28 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=abf4b7ab

AsyncScheduler: call_later asyncio compat (bug 591760)

Use call_later for asyncio compatibility. Also remove the
timeout_add method from SchedulerInterface, since there are
no more consumers.

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/AsyncScheduler.py | 9 +++--
 pym/portage/util/_async/SchedulerInterface.py | 1 -
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py 
b/pym/portage/util/_async/AsyncScheduler.py
index 90e803eba..c6b523eaa 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -65,6 +65,11 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
task.addExitListener(self._task_exit)
task.start()
 
+   if self._loadavg_check_id is not None:
+   self._loadavg_check_id.cancel()
+   self._loadavg_check_id = self._event_loop.call_later(
+   self._loadavg_latency, self._schedule)
+
# Triggers cleanup and exit listeners if there's nothing left 
to do.
self.poll()
 
@@ -80,14 +85,14 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
(self._max_jobs is True or self._max_jobs > 1):
# We have to schedule periodically, in case the load
# average has changed since the last call.
-   self._loadavg_check_id = self._event_loop.timeout_add(
+   self._loadavg_check_id = self._event_loop.call_later(
self._loadavg_latency, self._schedule)
self._schedule()
 
def _cleanup(self):
super(AsyncScheduler, self)._cleanup()
if self._loadavg_check_id is not None:
-   self._event_loop.source_remove(self._loadavg_check_id)
+   self._loadavg_check_id.cancel()
self._loadavg_check_id = None
 
def _async_wait(self):

diff --git a/pym/portage/util/_async/SchedulerInterface.py 
b/pym/portage/util/_async/SchedulerInterface.py
index ff39bc587..56b844616 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -15,7 +15,6 @@ class SchedulerInterface(SlotObject):
"IO_NVAL", "IO_OUT", "IO_PRI",
"io_add_watch",
"source_remove",
-   "timeout_add",
 
"add_reader",
"add_writer",

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

2018-04-29 Thread Zac Medico

commit: 542c6e6c20a1d93a3a2af47c8de50eac3c891d5d
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 29 20:48:23 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Apr 29 21:26:37 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=542c6e6c

MergeProcess: fix deprecated _set_returncode (bug 654276)

Move cleanup code from _set_returncode to _async_waitpid_cb,
since _set_returncode expects an os.waitpid return value which
is inconveniently different from the returncode that is passed to
asyncio.AbstractChildWatcher.add_child_handler callbacks.

Bug: https://bugs.gentoo.org/654276

 pym/portage/dbapi/_MergeProcess.py | 10 +++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py 
b/pym/portage/dbapi/_MergeProcess.py
index 35591af16..bfbe387e4 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2013 Gentoo Foundation
+# Copyright 2010-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import io
@@ -251,8 +251,12 @@ class MergeProcess(ForkProcess):
# in order to avoid a race condition.
os._exit(1)
 
-   def _set_returncode(self, wait_retval):
-   ForkProcess._set_returncode(self, wait_retval)
+   def _async_waitpid_cb(self, *args, **kwargs):
+   """
+   Override _async_waitpid_cb to perform cleanup that is
+   not necessarily idempotent.
+   """
+   ForkProcess._async_waitpid_cb(self, *args, **kwargs)
if self.returncode == portage.const.RETURNCODE_POSTINST_FAILURE:
self.postinst_failure = True
self.returncode = os.EX_OK

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-29 Thread Zac Medico

commit: be800bf0153a28ce034277d103a2021f93ac8b2e
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 29 05:10:27 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Apr 29 05:58:58 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=be800bf0

PopenProcess: add_child_handler asyncio compat (bug 591760)

Migrate to asyncio's AbstractChildWatcher.add_child_handler
interface.

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/PopenProcess.py | 31 +++
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/pym/portage/util/_async/PopenProcess.py 
b/pym/portage/util/_async/PopenProcess.py
index 4344b1c9d..3fb60d527 100644
--- a/pym/portage/util/_async/PopenProcess.py
+++ b/pym/portage/util/_async/PopenProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2017 Gentoo Foundation
+# Copyright 2012-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from _emerge.SubProcess import SubProcess
@@ -13,8 +13,7 @@ class PopenProcess(SubProcess):
self._registered = True
 
if self.pipe_reader is None:
-   self._reg_id = self.scheduler.child_watch_add(
-   self.pid, self._child_watch_cb)
+   self._async_waitpid()
else:
try:
self.pipe_reader.scheduler = self.scheduler
@@ -24,17 +23,25 @@ class PopenProcess(SubProcess):
self.pipe_reader.start()
 
def _pipe_reader_exit(self, pipe_reader):
-   self._reg_id = self.scheduler.child_watch_add(
-   self.pid, self._child_watch_cb)
+   self._async_waitpid()
 
-   def _child_watch_cb(self, pid, condition, user_data=None):
-   self._reg_id = None
-   self._waitpid_cb(pid, condition)
-   self.wait()
+   def _async_waitpid(self):
+   if self.returncode is None:
+   self.scheduler._asyncio_child_watcher.\
+   add_child_handler(self.pid, 
self._async_waitpid_cb)
+   else:
+   self._unregister()
+   self._async_wait()
 
-   def _set_returncode(self, wait_retval):
-   SubProcess._set_returncode(self, wait_retval)
+   def _async_waitpid_cb(self, pid, returncode):
if self.proc.returncode is None:
# Suppress warning messages like this:
# ResourceWarning: subprocess 1234 is still running
-   self.proc.returncode = self.returncode
+   self.proc.returncode = returncode
+   self._unregister()
+   self.returncode = returncode
+   self._async_wait()
+
+   def _poll(self):
+   # Simply rely on _async_waitpid_cb to set the returncode.
+   return self.returncode

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, pym/_emerge/, pym/portage/_emirrordist/

2018-04-28 Thread Zac Medico

commit: f01f504fc8d4ea7442379482889b1cee4be0fcf4
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 29 01:28:42 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Apr 29 04:31:45 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f01f504f

AsynchronousTask: remove unused _wait() methods (bug 653856)

Remove _wait() and _waitpid_loop() methods which are unused since
AsynchronousTask.wait() now uses run_until_complete(self.async_wait())
instead. Also remove the iteration method from SchedulerInterface,
since there are no more consumers.

Bug: https://bugs.gentoo.org/653856

 pym/_emerge/AbstractEbuildProcess.py| 27 
 pym/_emerge/AbstractPollTask.py |  8 +---
 pym/_emerge/AsynchronousLock.py |  9 
 pym/_emerge/AsynchronousTask.py |  6 ---
 pym/_emerge/CompositeTask.py| 48 +
 pym/_emerge/FifoIpcDaemon.py| 13 +-
 pym/_emerge/PipeReader.py   |  7 
 pym/_emerge/SpawnProcess.py | 12 +-
 pym/_emerge/SubProcess.py   | 56 +
 pym/portage/_emirrordist/MirrorDistTask.py  |  7 
 pym/portage/util/_async/AsyncScheduler.py   | 25 +--
 pym/portage/util/_async/AsyncTaskFuture.py  |  8 
 pym/portage/util/_async/PipeLogger.py   |  7 
 pym/portage/util/_async/PipeReaderBlockingIO.py | 10 -
 pym/portage/util/_async/SchedulerInterface.py   |  1 -
 15 files changed, 6 insertions(+), 238 deletions(-)

diff --git a/pym/_emerge/AbstractEbuildProcess.py 
b/pym/_emerge/AbstractEbuildProcess.py
index b10aa4bfa..03c834912 100644
--- a/pym/_emerge/AbstractEbuildProcess.py
+++ b/pym/_emerge/AbstractEbuildProcess.py
@@ -386,33 +386,6 @@ class AbstractEbuildProcess(SpawnProcess):
if not self.cancelled:
self._unexpected_exit()
 
-   def _wait(self):
-   """
-   Override _wait to unlock self._build_dir if necessary. 
Normally, it
-   should already be unlocked, so this functions only as a 
failsafe.
-   Execution of the failsafe code will automatically become a fatal
-   error at the same time as event loop recursion is disabled.
-   """
-   # SpawnProcess._wait() requires the pid, so wait here for the
-   # pid to become available.
-   while self._start_future is not None:
-   self.scheduler.run_until_complete(self._start_future)
-
-   SpawnProcess._wait(self)
-
-   if self._build_dir is not None:
-   self._build_dir_unlock = self._build_dir.async_unlock()
-   # Unlock only once.
-   self._build_dir = None
-
-   if not (self._build_dir_unlock is None or
-   self._build_dir_unlock.done()):
-   # This will automatically become a fatal error at the 
same
-   # time as event loop recursion is disabled.
-   
self.scheduler.run_until_complete(self._build_dir_unlock)
-
-   return self.returncode
-
def _async_wait(self):
"""
Override _async_wait to asynchronously unlock self._build_dir

diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py
index 0aac97be5..0ce3594b4 100644
--- a/pym/_emerge/AbstractPollTask.py
+++ b/pym/_emerge/AbstractPollTask.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2015 Gentoo Foundation
+# Copyright 1999-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import array
@@ -136,12 +136,6 @@ class AbstractPollTask(AsynchronousTask):
self.returncode = self.returncode or os.EX_OK
self._async_wait()
 
-   def _wait(self):
-   if self.returncode is not None:
-   return self.returncode
-   self._wait_loop()
-   return self.returncode
-
def _wait_loop(self, timeout=None):
loop = getattr(self.scheduler, '_asyncio_wrapper', 
self.scheduler)
tasks = [self.async_wait()]

diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py
index 2019adaab..c5991bcff 100644
--- a/pym/_emerge/AsynchronousLock.py
+++ b/pym/_emerge/AsynchronousLock.py
@@ -82,15 +82,6 @@ class AsynchronousLock(AsynchronousTask):
self._imp.poll()
return self.returncode
 
-   def _wait(self):
-   """
-   Deprecated. Use _async_wait() instead.
-   """
-   if self.returncode is not None:
-   return self.returncode
-   self.returncode = self._imp.wait()
-

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-28 Thread Zac Medico

commit: 2ab6eed08c9eafa29de6100b19c9c072b17b9167
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 29 04:06:25 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Apr 29 04:06:25 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2ab6eed0

PipeReaderBlockingIO._reader_thread: asyncio compat (bug 653856)

Use call_soon_threadsafe for asyncio compatibility, and also
remove idle_add from SchedulerInterface since there are no more
consumers.

Bug: https://bugs.gentoo.org/653856

 pym/portage/util/_async/PipeReaderBlockingIO.py | 3 +--
 pym/portage/util/_async/SchedulerInterface.py   | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/pym/portage/util/_async/PipeReaderBlockingIO.py 
b/pym/portage/util/_async/PipeReaderBlockingIO.py
index b426f7695..0727d2c6a 100644
--- a/pym/portage/util/_async/PipeReaderBlockingIO.py
+++ b/pym/portage/util/_async/PipeReaderBlockingIO.py
@@ -55,7 +55,7 @@ class PipeReaderBlockingIO(AbstractPollTask):
del self._threads[f]
if not self._threads:
# Thread-safe callback to 
EventLoop
-   
self.scheduler.idle_add(self._eof)
+   
self.scheduler.call_soon_threadsafe(self._eof)
break
f.close()
 
@@ -64,7 +64,6 @@ class PipeReaderBlockingIO(AbstractPollTask):
if self.returncode is None:
self.returncode = os.EX_OK
self._async_wait()
-   return False
 
def _cancel(self):
self._terminate.set()

diff --git a/pym/portage/util/_async/SchedulerInterface.py 
b/pym/portage/util/_async/SchedulerInterface.py
index ab2a70852..acacf7c31 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2013 Gentoo Foundation
+# Copyright 2012-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import gzip
@@ -14,7 +14,6 @@ class SchedulerInterface(SlotObject):
_event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN",
"IO_NVAL", "IO_OUT", "IO_PRI",
"child_watch_add",
-   "idle_add",
"io_add_watch",
"iteration",
"source_remove",

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-28 Thread Zac Medico

commit: 532366feaf8e3cb04e0d19d90498cfe0b9361ab1
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Apr 28 23:27:49 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Apr 28 23:27:49 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=532366fe

PipeReaderBlockingIO: fix deprecated _wait usage (bug 653856)

Use _async_wait() instead of wait().

Bug: https://bugs.gentoo.org/653856

 pym/portage/util/_async/PipeReaderBlockingIO.py | 9 ++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/pym/portage/util/_async/PipeReaderBlockingIO.py 
b/pym/portage/util/_async/PipeReaderBlockingIO.py
index b06adf6ed..b426f7695 100644
--- a/pym/portage/util/_async/PipeReaderBlockingIO.py
+++ b/pym/portage/util/_async/PipeReaderBlockingIO.py
@@ -1,4 +1,4 @@
-# Copyright 2012 Gentoo Foundation
+# Copyright 2012-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 try:
@@ -63,7 +63,7 @@ class PipeReaderBlockingIO(AbstractPollTask):
self._registered = False
if self.returncode is None:
self.returncode = os.EX_OK
-   self.wait()
+   self._async_wait()
return False
 
def _cancel(self):
@@ -71,9 +71,12 @@ class PipeReaderBlockingIO(AbstractPollTask):
self._registered = False
if self.returncode is None:
self.returncode = self._cancelled_returncode
-   self.wait()
+   self._async_wait()
 
def _wait(self):
+   """
+   Deprecated. Use _async_wait() instead.
+   """
if self.returncode is not None:
return self.returncode
self._wait_loop()

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-28 Thread Zac Medico

commit: 98e5852117cbda19b4f4266eb71a053dced64a88
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Apr 28 23:17:30 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Apr 28 23:17:30 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=98e58521

AsyncTaskFuture: fix deprecated _wait usage (bug 653856)

Fix _done_callback to use _async_wait() instead of wait().

Bug: https://bugs.gentoo.org/653856

 pym/portage/util/_async/AsyncTaskFuture.py | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/portage/util/_async/AsyncTaskFuture.py 
b/pym/portage/util/_async/AsyncTaskFuture.py
index ce881ba8d..cdc842b05 100644
--- a/pym/portage/util/_async/AsyncTaskFuture.py
+++ b/pym/portage/util/_async/AsyncTaskFuture.py
@@ -28,9 +28,12 @@ class AsyncTaskFuture(AsynchronousTask):
self.returncode = os.EX_OK
else:
self.returncode = 1
-   self.wait()
+   self._async_wait()
 
def _wait(self):
+   """
+   Deprecated. Use _async_wait() instead.
+   """
if self.returncode is None:
self.scheduler.run_until_complete(self.future)
return self.returncode

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

2018-04-28 Thread Zac Medico

commit: 1b0a408fcd556b7d9912856b852a331260d22022
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Apr 28 22:48:51 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Apr 28 22:48:51 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1b0a408f

MirrorDistTask: fix super in _async_wait

Fixes: 27801746e9aa ("MirrorDistTask: fix deprecated _wait usage (bug 653856)")

 pym/portage/_emirrordist/MirrorDistTask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/_emirrordist/MirrorDistTask.py 
b/pym/portage/_emirrordist/MirrorDistTask.py
index a0912d673..b8cde0af4 100644
--- a/pym/portage/_emirrordist/MirrorDistTask.py
+++ b/pym/portage/_emirrordist/MirrorDistTask.py
@@ -253,4 +253,4 @@ class MirrorDistTask(CompositeTask):
Override _async_wait to call self._cleanup().
"""
self._cleanup()
-   super(CompositeTask, self)._async_wait()
+   super(MirrorDistTask, self)._async_wait()

[gentoo-commits] proj/portage:master commit in: pym/portage/

2018-04-28 Thread Zac Medico

commit: 97887768e3f5e8bc1b520b119fde97fed16cfe7d
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Apr 28 22:59:21 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Apr 28 23:05:25 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=97887768

PortageKeyError: fix __unicode__ AttributeError under python2.7

  File "pym/portage/exception.py", line 32, in __unicode__
if isinstance(self.value, unicode):
AttributeError: 'PortageKeyError' object has no attribute 'value'

 pym/portage/exception.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pym/portage/exception.py b/pym/portage/exception.py
index 263cdf087..aed8beeb9 100644
--- a/pym/portage/exception.py
+++ b/pym/portage/exception.py
@@ -44,6 +44,9 @@ class PortageException(Exception):
 
 class PortageKeyError(KeyError, PortageException):
__doc__ = KeyError.__doc__
+   def __init__(self, value):
+   KeyError.__init__(self, value)
+   PortageException.__init__(self, value)
 
 class CorruptionError(PortageException):
"""Corruption indication"""

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-28 Thread Zac Medico

commit: 3d81af6440cfd20cd007568779d6129083f1de24
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Apr 28 22:45:56 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Apr 28 22:45:56 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d81af64

AsyncScheduler: fix deprecated _wait usage (bug 653856)

Override AsynchronousTask._async_wait() for cleanup, since
AsynchronousTask._wait() is deprecated.

Bug: https://bugs.gentoo.org/653856

 pym/portage/util/_async/AsyncScheduler.py | 13 +++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py 
b/pym/portage/util/_async/AsyncScheduler.py
index 1b870c771..2ddbaa885 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -29,12 +29,11 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 
def _poll(self):
if not (self._is_work_scheduled() or self._keep_scheduling()):
-   self._cleanup()
-
if self._error_count > 0:
self.returncode = 1
else:
self.returncode = os.EX_OK
+   self._async_wait()
return self.returncode
 
def _cancel(self):
@@ -91,7 +90,17 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
self._event_loop.source_remove(self._loadavg_check_id)
self._loadavg_check_id = None
 
+   def _async_wait(self):
+   """
+   Override _async_wait to call self._cleanup().
+   """
+   self._cleanup()
+   super(AsyncScheduler, self)._async_wait()
+
def _wait(self):
+   """
+   Deprecated. Use _async_wait() instead.
+   """
# Loop while there are jobs to be scheduled.
while self._keep_scheduling():
self._event_loop.iteration()

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

2018-04-28 Thread Zac Medico

commit: 27801746e9aa7e9ffc02408b2b005aa106c0d8a7
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Apr 28 21:51:01 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Apr 28 21:55:46 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=27801746

MirrorDistTask: fix deprecated _wait usage (bug 653856)

Override AsynchronousTask._async_wait() for cleanup, since
AsynchronousTask._wait() is deprecated.

Bug: https://bugs.gentoo.org/653856

 pym/portage/_emirrordist/MirrorDistTask.py | 18 ++
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/pym/portage/_emirrordist/MirrorDistTask.py 
b/pym/portage/_emirrordist/MirrorDistTask.py
index 48d0f7cf2..a0912d673 100644
--- a/pym/portage/_emirrordist/MirrorDistTask.py
+++ b/pym/portage/_emirrordist/MirrorDistTask.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2014 Gentoo Foundation
+# Copyright 2013-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -46,7 +46,7 @@ class MirrorDistTask(CompositeTask):
 
self._assert_current(fetch)
if self._was_cancelled():
-   self.wait()
+   self._async_wait()
return
 
if self._config.options.delete:
@@ -63,7 +63,7 @@ class MirrorDistTask(CompositeTask):
 
self._assert_current(deletion)
if self._was_cancelled():
-   self.wait()
+   self._async_wait()
return
 
self._post_deletion()
@@ -80,7 +80,7 @@ class MirrorDistTask(CompositeTask):
 
self.returncode = os.EX_OK
self._current_task = None
-   self.wait()
+   self._async_wait()
 
def _update_recycle_db(self):
 
@@ -242,5 +242,15 @@ class MirrorDistTask(CompositeTask):
self._async_wait()
 
def _wait(self):
+   """
+   Deprecated. Use _async_wait() instead.
+   """
CompositeTask._wait(self)
self._cleanup()
+
+   def _async_wait(self):
+   """
+   Override _async_wait to call self._cleanup().
+   """
+   self._cleanup()
+   super(CompositeTask, self)._async_wait()

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

2018-04-28 Thread Zac Medico

commit: 7c652d1b967f9c4c6a7fbd9fc5e46a0e57438a16
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Apr 25 06:42:10 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Apr 28 13:56:42 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c652d1b

FetchIterator: fix event loop recursion (bug 654038)

Since construction of FetchTask instances requires results from
aux_get calls that would trigger event loop recursion when executed
synchronously, add an _async_fetch_tasks function to construct
FetchTask instances asynchronously and return a Future. Use an
_EbuildFetchTasks class to wait for the FetchTask instances to
become available, and then execute them.

Bug: https://bugs.gentoo.org/654038

 pym/portage/_emirrordist/FetchIterator.py | 324 --
 1 file changed, 218 insertions(+), 106 deletions(-)

diff --git a/pym/portage/_emirrordist/FetchIterator.py 
b/pym/portage/_emirrordist/FetchIterator.py
index 38419799d..366453c12 100644
--- a/pym/portage/_emirrordist/FetchIterator.py
+++ b/pym/portage/_emirrordist/FetchIterator.py
@@ -1,4 +1,4 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import threading
@@ -7,14 +7,18 @@ from portage import os
 from portage.checksum import (_apply_hash_filter,
_filter_unaccelarated_hashes, _hash_filter)
 from portage.dep import use_reduce
-from portage.exception import PortageException
+from portage.exception import PortageException, PortageKeyError
+from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._async.TaskScheduler import TaskScheduler
+from portage.util.futures.iter_completed import iter_gather
 from .FetchTask import FetchTask
+from _emerge.CompositeTask import CompositeTask
+
 
 class FetchIterator(object):
 
def __init__(self, config):
self._config = config
-   self._log_failure = config.log_failure
self._terminated = threading.Event()
 
def terminate(self):
@@ -41,9 +45,6 @@ class FetchIterator(object):
 
portdb = self._config.portdb
get_repo_for_location = 
portdb.repositories.get_repo_for_location
-   file_owners = self._config.file_owners
-   file_failures = self._config.file_failures
-   restrict_mirror_exemptions = 
self._config.restrict_mirror_exemptions
 
hash_filter = _hash_filter(
portdb.settings.get("PORTAGE_CHECKSUM_FILTER", ""))
@@ -59,110 +60,221 @@ class FetchIterator(object):
 
# Reset state so the Manifest is pulled once
# for this cp / tree combination.
-   digests = None
repo_config = get_repo_for_location(tree)
+   digests_future = 
portdb._event_loop.create_future()
 
for cpv in portdb.cp_list(cp, mytree=tree):
 
if self._terminated.is_set():
return
 
-   try:
-   restrict, = portdb.aux_get(cpv, 
("RESTRICT",),
-   mytree=tree)
-   except (KeyError, PortageException) as 
e:
-   
self._log_failure("%s\t\taux_get exception %s" %
-   (cpv, e))
-   continue
-
-   # Here we use matchnone=True to ignore 
conditional parts
-   # of RESTRICT since they don't apply 
unconditionally.
-   # Assume such conditionals only apply 
on the client side.
-   try:
-   restrict = 
frozenset(use_reduce(restrict,
-   flat=True, 
matchnone=True))
-   except PortageException as e:
-   
self._log_failure("%s\t\tuse_reduce exception %s" %
-   (cpv, e))
-   continue
-
-   if "fetch" in restrict:
-   continue
-
-   try:
-   uri_map = 
portdb.getFetchMap(cpv)
-   except PortageException as e:
-   
self._log_failure("%s\t\tgetFetchMap exception %s" %
-

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/dbapi/, pym/portage/package/ebuild/_parallel_manifest/, ...

2018-04-27 Thread Zac Medico

commit: 3e77f0199cb401acf974089fb6aa378fd45d0e90
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Apr 24 06:54:05 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Fri Apr 27 22:56:02 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3e77f019

ManifestScheduler: async fetchlist_dict (bug 653946)

In order to avoid event loop recursion, pass fetchlist_dict to
ManifestTask as a Future.

Bug: https://bugs.gentoo.org/653946

 pym/portage/dbapi/porttree.py  | 70 ++
 .../ebuild/_parallel_manifest/ManifestScheduler.py | 25 
 .../ebuild/_parallel_manifest/ManifestTask.py  | 24 +++-
 pym/portage/tests/dbapi/test_portdb_cache.py   |  3 +-
 4 files changed, 105 insertions(+), 17 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 975f03d5e..3ce214cd7 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -37,6 +37,7 @@ from portage import _unicode_encode
 from portage import OrderedDict
 from portage.util._eventloop.EventLoop import EventLoop
 from portage.util._eventloop.global_event_loop import global_event_loop
+from portage.util.futures.iter_completed import iter_gather
 from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
 
 import os as _os
@@ -1393,6 +1394,75 @@ class FetchlistDict(Mapping):
if sys.hexversion >= 0x300:
keys = __iter__
 
+
+def _async_manifest_fetchlist(portdb, repo_config, cp, cpv_list=None,
+   max_jobs=None, max_load=None, loop=None):
+   """
+   Asynchronous form of FetchlistDict, with max_jobs and max_load
+   parameters in order to control async_aux_get concurrency.
+
+   @param portdb: portdbapi instance
+   @type portdb: portdbapi
+   @param repo_config: repository configuration for a Manifest
+   @type repo_config: RepoConfig
+   @param cp: cp for a Manifest
+   @type cp: str
+   @param cpv_list: list of ebuild cpv values for a Manifest
+   @type cpv_list: list
+   @param max_jobs: max number of futures to process concurrently (default
+   is multiprocessing.cpu_count())
+   @type max_jobs: int
+   @param max_load: max load allowed when scheduling a new future,
+   otherwise schedule no more than 1 future at a time (default
+   is multiprocessing.cpu_count())
+   @type max_load: int or float
+   @param loop: event loop
+   @type loop: EventLoop
+   @return: a Future resulting in a Mapping compatible with FetchlistDict
+   @rtype: asyncio.Future (or compatible)
+   """
+   loop = loop or global_event_loop()
+   loop = getattr(loop, '_asyncio_wrapper', loop)
+   result = loop.create_future()
+   cpv_list = (portdb.cp_list(cp, mytree=repo_config.location)
+   if cpv_list is None else cpv_list)
+
+   def gather_done(gather_result):
+   # All exceptions must be consumed from gather_result before this
+   # function returns, in order to avoid triggering the event 
loop's
+   # exception handler.
+   e = None
+   if not gather_result.cancelled():
+   for future in gather_result.result():
+   if (future.done() and not future.cancelled() and
+   future.exception() is not None):
+   e = future.exception()
+
+   if result.cancelled():
+   return
+   elif e is None:
+   result.set_result(dict((k, list(v.result()))
+   for k, v in zip(cpv_list, 
gather_result.result(
+   else:
+   result.set_exception(e)
+
+   gather_result = iter_gather(
+   # Use a generator expression for lazy evaluation, so that 
iter_gather
+   # controls the number of concurrent async_fetch_map calls.
+   (portdb.async_fetch_map(cpv, mytree=repo_config.location, 
loop=loop)
+   for cpv in cpv_list),
+   max_jobs=max_jobs,
+   max_load=max_load,
+   loop=loop,
+   )
+
+   gather_result.add_done_callback(gather_done)
+   result.add_done_callback(lambda result:
+   gather_result.cancel() if result.cancelled() else None)
+
+   return result
+
+
 def _parse_uri_map(cpv, metadata, use=None):
 
myuris = use_reduce(metadata.get('SRC_URI', ''),

diff --git a/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py 
b/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py
index 38ac4825e..fabea9bc1 100644
--- a/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py
+++ b/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py
@@ -1,10 +1,10 @@
-# Copyright 2012-2013 Gentoo Foundation
+# Copyright

[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/

2018-04-27 Thread Zac Medico

commit: be61882996099322bb3a1e82e71f475b4141ad40
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Apr 24 23:28:08 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Fri Apr 27 21:33:00 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=be618829

Add iter_gather function (bug 653946)

This is similar to asyncio.gather, but takes an iterator of
futures as input, and includes support for max_jobs and max_load
parameters. For bug 653946, this will be used to asynchronously
gather the results of the portdbapi.async_fetch_map calls that
are required to generate a Manifest, while using the max_jobs
parameter to limit the number of concurrent async_aux_get calls.

Bug: https://bugs.gentoo.org/653946

 pym/portage/util/futures/iter_completed.py | 73 ++
 1 file changed, 73 insertions(+)

diff --git a/pym/portage/util/futures/iter_completed.py 
b/pym/portage/util/futures/iter_completed.py
index 5ad075305..1d6a9a4bd 100644
--- a/pym/portage/util/futures/iter_completed.py
+++ b/pym/portage/util/futures/iter_completed.py
@@ -112,3 +112,76 @@ def async_iter_completed(futures, max_jobs=None, 
max_load=None, loop=None):
# cleanup in case of interruption by SIGINT, etc
scheduler.cancel()
scheduler.wait()
+
+
+def iter_gather(futures, max_jobs=None, max_load=None, loop=None):
+   """
+   This is similar to asyncio.gather, but takes an iterator of
+   futures as input, and includes support for max_jobs and max_load
+   parameters.
+
+   @param futures: iterator of asyncio.Future (or compatible)
+   @type futures: iterator
+   @param max_jobs: max number of futures to process concurrently (default
+   is multiprocessing.cpu_count())
+   @type max_jobs: int
+   @param max_load: max load allowed when scheduling a new future,
+   otherwise schedule no more than 1 future at a time (default
+   is multiprocessing.cpu_count())
+   @type max_load: int or float
+   @param loop: event loop
+   @type loop: EventLoop
+   @return: a Future resulting in a list of done input futures, in the
+   same order that they were yielded from the input iterator
+   @rtype: asyncio.Future (or compatible)
+   """
+   loop = loop or global_event_loop()
+   loop = getattr(loop, '_asyncio_wrapper', loop)
+   result = loop.create_future()
+   futures_list = []
+
+   def future_generator():
+   for future in futures:
+   futures_list.append(future)
+   yield future
+
+   completed_iter = async_iter_completed(
+   future_generator(),
+   max_jobs=max_jobs,
+   max_load=max_load,
+   loop=loop,
+   )
+
+   def handle_result(future_done_set):
+   if result.cancelled():
+   if not future_done_set.cancelled():
+   # All exceptions must be consumed from 
future_done_set, in order
+   # to avoid triggering the event loop's 
exception handler.
+   list(future.exception() for future in 
future_done_set.result()
+   if not future.cancelled())
+   return
+
+   try:
+   handle_result.current_task = next(completed_iter)
+   except StopIteration:
+   result.set_result(futures_list)
+   else:
+   
handle_result.current_task.add_done_callback(handle_result)
+
+   try:
+   handle_result.current_task = next(completed_iter)
+   except StopIteration:
+   handle_result.current_task = None
+   result.set_result(futures_list)
+   else:
+   handle_result.current_task.add_done_callback(handle_result)
+
+   def cancel_callback(result):
+   if (result.cancelled() and
+   handle_result.current_task is not None and
+   not handle_result.current_task.done()):
+   handle_result.current_task.cancel()
+
+   result.add_done_callback(cancel_callback)
+
+   return result

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-26 Thread Zac Medico

commit: a65d93bdeb4d964603cd0ce9b0a75a571c9bdefa
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Apr 26 09:19:22 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Apr 26 09:31:09 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a65d93bd

SchedulerInterface: expose AbstractEventLoop methods (bug 591760)

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/SchedulerInterface.py | 34 +++
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/pym/portage/util/_async/SchedulerInterface.py 
b/pym/portage/util/_async/SchedulerInterface.py
index 21420ae41..ab2a70852 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -13,10 +13,36 @@ class SchedulerInterface(SlotObject):
 
_event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN",
"IO_NVAL", "IO_OUT", "IO_PRI",
-   "call_soon", "call_soon_threadsafe",
-   "child_watch_add", "create_future",
-   "idle_add", "io_add_watch", "iteration", "run_until_complete",
-   "source_remove", "timeout_add")
+   "child_watch_add",
+   "idle_add",
+   "io_add_watch",
+   "iteration",
+   "source_remove",
+   "timeout_add",
+
+   "add_reader",
+   "add_writer",
+   "call_at",
+   "call_exception_handler",
+   "call_later",
+   "call_soon",
+   "call_soon_threadsafe",
+   "close",
+   "create_future",
+   "default_exception_handler",
+   "get_debug",
+   "is_closed",
+   "is_running",
+   "remove_reader",
+   "remove_writer",
+   "run_in_executor",
+   "run_until_complete",
+   "set_debug",
+   "time",
+
+   "_asyncio_child_watcher",
+   "_asyncio_wrapper",
+   )
 
__slots__ = _event_loop_attrs + ("_event_loop", "_is_background")

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

2018-04-26 Thread Zac Medico

commit: 391daef6fce981acd5a01e41a0f7238044c48877
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Apr 26 08:29:04 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Apr 26 08:43:53 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=391daef6

MirrorDistTask._term_callback(): use _async_wait() (bug 591760)

Use _async_wait() to avoid event loop recursion, but don't call
it prematurely, since since that could trigger event loop recursion
if the current (cancelled) task's exit callback does not set the
returncode first.

Bug: https://bugs.gentoo.org/591760

 pym/portage/_emirrordist/MirrorDistTask.py | 10 +-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/pym/portage/_emirrordist/MirrorDistTask.py 
b/pym/portage/_emirrordist/MirrorDistTask.py
index a34f2c061..48d0f7cf2 100644
--- a/pym/portage/_emirrordist/MirrorDistTask.py
+++ b/pym/portage/_emirrordist/MirrorDistTask.py
@@ -231,7 +231,15 @@ class MirrorDistTask(CompositeTask):
if self._fetch_iterator is not None:
self._fetch_iterator.terminate()
self.cancel()
-   self.wait()
+   if self.returncode is None:
+   # In this case, the exit callback for 
self._current_task will
+   # trigger notification of exit listeners. Don't call 
_async_wait()
+   # yet, since that could trigger event loop recursion if 
the
+   # current (cancelled) task's exit callback does not set 
the
+   # returncode first.
+   pass
+   else:
+   self._async_wait()
 
def _wait(self):
CompositeTask._wait(self)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-26 Thread Zac Medico

commit: d16206ccd315edbd4bd7a46c0b1aa9e59a46db43
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Apr 26 07:45:32 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Apr 26 08:02:28 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d16206cc

AsyncScheduler._poll(): fix event loop recursion (bug 591760)

Call self._cleanup() and set the returncode, in order to avoid
event loop recursion in self.wait().

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/AsyncScheduler.py | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py 
b/pym/portage/util/_async/AsyncScheduler.py
index b89b57dab..1b870c771 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -29,7 +29,12 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 
def _poll(self):
if not (self._is_work_scheduled() or self._keep_scheduling()):
-   self.wait()
+   self._cleanup()
+
+   if self._error_count > 0:
+   self.returncode = 1
+   else:
+   self.returncode = os.EX_OK
return self.returncode
 
def _cancel(self):

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-25 Thread Zac Medico

commit: 384b61dd24a21aaccd7c643e58b07ea56c3cbdfc
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Apr 26 05:31:54 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Apr 26 05:46:30 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=384b61dd

PipeLogger._output_handler: use _async_wait() (bug 591760)

Use _async_wait() to avoid event loop recursion.

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/PipeLogger.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/portage/util/_async/PipeLogger.py 
b/pym/portage/util/_async/PipeLogger.py
index aa605d94d..ed1202edf 100644
--- a/pym/portage/util/_async/PipeLogger.py
+++ b/pym/portage/util/_async/PipeLogger.py
@@ -1,4 +1,4 @@
-# Copyright 2008-2013 Gentoo Foundation
+# Copyright 2008-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import fcntl
@@ -89,7 +89,8 @@ class PipeLogger(AbstractPollTask):
if not buf:
# EOF
self._unregister()
-   self.wait()
+   self.returncode = self.returncode or os.EX_OK
+   self._async_wait()
break
 
else:

[gentoo-commits] proj/portage:master commit in: pym/portage/util/, pym/portage/util/_async/, pym/_emerge/

2018-04-25 Thread Zac Medico

commit: 6c865d6b3130de63dc3d534d2a0596dd59d5e662
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Apr 24 02:21:59 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Apr 26 03:19:21 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c865d6b

AsynchronousTask: add scheduler attribute (bug 653856)

Add an AsynchronousTask.scheduler attribute, which will hold an event
loop instance in order to work with Future instances. Name the
attribute "scheduler" because this is the name used by many existing
subclasses of AsynchronousTask. Since the AsyncScheduler class already
has an _event_loop attribute that it inherits from PollScheduler, use
a property for compatibility. Also update the SlotObject constructor
to allow a subclass to override an attribute from __slots__ with a
property, so that AsyncScheduler can use a property for the scheduler
attribute.

Bug: https://bugs.gentoo.org/653856

 pym/_emerge/AbstractPollTask.py| 3 +--
 pym/_emerge/AsynchronousLock.py| 2 +-
 pym/_emerge/AsynchronousTask.py| 2 +-
 pym/_emerge/CompositeTask.py   | 2 +-
 pym/portage/util/SlotObject.py | 9 -
 pym/portage/util/_async/AsyncScheduler.py  | 7 +++
 pym/portage/util/_async/AsyncTaskFuture.py | 2 +-
 7 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py
index 0d38bd481..aa8fc6593 100644
--- a/pym/_emerge/AbstractPollTask.py
+++ b/pym/_emerge/AbstractPollTask.py
@@ -11,8 +11,7 @@ from _emerge.AsynchronousTask import AsynchronousTask
 
 class AbstractPollTask(AsynchronousTask):
 
-   __slots__ = ("scheduler",) + \
-   ("_registered",)
+   __slots__ = ("_registered",)
 
_bufsize = 4096
 

diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py
index 78ab37ecb..0178feab2 100644
--- a/pym/_emerge/AsynchronousLock.py
+++ b/pym/_emerge/AsynchronousLock.py
@@ -37,7 +37,7 @@ class AsynchronousLock(AsynchronousTask):
signals to the main thread).
"""
 
-   __slots__ = ('path', 'scheduler',) + \
+   __slots__ = ('path',) + \
('_imp', '_force_async', '_force_dummy', '_force_process', \
'_force_thread', '_unlock_future')
 

diff --git a/pym/_emerge/AsynchronousTask.py b/pym/_emerge/AsynchronousTask.py
index da58261db..e29324440 100644
--- a/pym/_emerge/AsynchronousTask.py
+++ b/pym/_emerge/AsynchronousTask.py
@@ -16,7 +16,7 @@ class AsynchronousTask(SlotObject):
the task is complete and self.returncode has been set.
"""
 
-   __slots__ = ("background", "cancelled", "returncode") + \
+   __slots__ = ("background", "cancelled", "returncode", "scheduler") + \
("_exit_listeners", "_exit_listener_stack", "_start_listeners",
"_waiting")
 

diff --git a/pym/_emerge/CompositeTask.py b/pym/_emerge/CompositeTask.py
index f3acc9696..bfd4bacbd 100644
--- a/pym/_emerge/CompositeTask.py
+++ b/pym/_emerge/CompositeTask.py
@@ -6,7 +6,7 @@ from portage import os
 
 class CompositeTask(AsynchronousTask):
 
-   __slots__ = ("scheduler",) + ("_current_task",)
+   __slots__ = ("_current_task",)
 
_TASK_QUEUED = -1
 

diff --git a/pym/portage/util/SlotObject.py b/pym/portage/util/SlotObject.py
index 4bb682258..ba6215874 100644
--- a/pym/portage/util/SlotObject.py
+++ b/pym/portage/util/SlotObject.py
@@ -20,7 +20,14 @@ class SlotObject(object):
raise AssertionError(
"class '%s' duplicates '%s' 
value in __slots__ of base class '%s'" %
(self.__class__.__name__, 
myattr, c.__name__))
-   setattr(self, myattr, myvalue)
+   try:
+   setattr(self, myattr, myvalue)
+   except AttributeError:
+   # Allow a property to override a 
__slots__ value, but raise an
+   # error if the intended value is 
something other than None.
+   if not (myvalue is None and
+   isinstance(getattr(type(self), 
myattr, None), property)):
+   raise
 
if kwargs:
raise TypeError(

diff --git a/pym/portage/util/_async/AsyncScheduler.py 
b/pym/portage/util/_async/AsyncScheduler.py
index 9beb8a848..b89b57dab 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -20,6 +20,13 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
self._remaining_tasks = True
self._loadavg_check_id = None
 
+   @property
+   def scheduler(self):
+   """
+

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/ebuild/, pym/_emerge/

2018-04-25 Thread Zac Medico

commit: 71a5a82313226f7be0d966d49392a53139a96f6b
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Apr 24 02:47:11 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Apr 26 03:19:22 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=71a5a823

AsynchronousTask: add async_wait() method (bug 653856)

Since the AsynchronousTask.wait() method is prone to event loop
recursion, deprecate it, and add an async_wait() method method to
replace it. Instead of using task.wait() in order to implicitly run
the event loop, now loop.run_until_complete(task.async_wait()) will
be used to explicitly run the event loop. This explicit approach will
make it more obvious when code will trigger event loop recursion
which would not be compatible with asyncio's default event loop.

Bug: https://bugs.gentoo.org/653856

 pym/_emerge/AsynchronousTask.py | 23 +++
 pym/portage/tests/ebuild/test_ipc_daemon.py |  2 +-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/pym/_emerge/AsynchronousTask.py b/pym/_emerge/AsynchronousTask.py
index e29324440..7d2e6253b 100644
--- a/pym/_emerge/AsynchronousTask.py
+++ b/pym/_emerge/AsynchronousTask.py
@@ -29,6 +29,26 @@ class AsynchronousTask(SlotObject):
self._start_hook()
self._start()
 
+   def async_wait(self):
+   """
+   Wait for returncode asynchronously. Notification is available
+   via the add_done_callback method of the returned Future 
instance.
+
+   @returns: Future, result is self.returncode
+   """
+   waiter = self.scheduler.create_future()
+   exit_listener = lambda self: waiter.set_result(self.returncode)
+   self.addExitListener(exit_listener)
+   waiter.add_done_callback(lambda waiter:
+   self.removeExitListener(exit_listener) if 
waiter.cancelled() else None)
+   if self.returncode is not None:
+   # If the returncode is not None, it means the exit 
event has already
+   # happened, so use _async_wait() to guarantee that the 
exit_listener
+   # is called. This does not do any harm because a given 
exit listener
+   # is never called more than once.
+   self._async_wait()
+   return waiter
+
def _start(self):
self.returncode = os.EX_OK
self.wait()
@@ -47,6 +67,9 @@ class AsynchronousTask(SlotObject):
return self.returncode
 
def wait(self):
+   """
+   Deprecated. Use async_wait() instead.
+   """
if self.returncode is None:
if not self._waiting:
self._waiting = True

diff --git a/pym/portage/tests/ebuild/test_ipc_daemon.py 
b/pym/portage/tests/ebuild/test_ipc_daemon.py
index bc18cdf64..e6da51a76 100644
--- a/pym/portage/tests/ebuild/test_ipc_daemon.py
+++ b/pym/portage/tests/ebuild/test_ipc_daemon.py
@@ -157,6 +157,6 @@ class IpcDaemonTestCase(TestCase):
try:
task_scheduler.start()
event_loop.run_until_complete(self._run_done)
-   task_scheduler.wait()
+   
event_loop.run_until_complete(task_scheduler.async_wait())
finally:
timeout_handle.cancel()

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

2018-04-25 Thread Zac Medico

commit: a7c8ddd9968741fd51ba21500a9f6b9ef27bef15
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Apr 25 07:28:44 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Apr 25 07:28:44 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7c8ddd9

FetchTask._fetch_uri: use _async_wait() (bug 591760)

Use _async_wait() to avoid event loop recursion in
emirrordist --dry-run mode.

Bug: https://bugs.gentoo.org/591760

 pym/portage/_emirrordist/FetchTask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/_emirrordist/FetchTask.py 
b/pym/portage/_emirrordist/FetchTask.py
index 47908cb6b..1440b697c 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -404,7 +404,7 @@ class FetchTask(CompositeTask):
(self.distfile, uri))
self._success()
self.returncode = os.EX_OK
-   self.wait()
+   self._async_wait()
return
 
if self.config.options.temp_dir:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

2018-04-24 Thread Zac Medico

commit: 698bbcc494e06719cdbb0522aff6a5fb0d249b36
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Apr 24 06:31:32 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Apr 24 06:55:20 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=698bbcc4

portdbapi.async_fetch_map: fix future.exception() reference

Fixes: 8002d9343c38 ("portdbapi: add async_fetch_map method (bug 653810)")

 pym/portage/dbapi/porttree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 8a5fc690c..975f03d5e 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -759,7 +759,7 @@ class portdbapi(dbapi):
if result.cancelled():
return
if aux_get_future.exception() is not None:
-   if isinstance(future.exception(), 
PortageKeyError):
+   if isinstance(aux_get_future.exception(), 
PortageKeyError):
# Convert this to an 
InvalidDependString exception since
# callers already handle it.

result.set_exception(portage.exception.InvalidDependString(

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

2018-04-23 Thread Zac Medico

commit: 508c7667bb0755c9f72b443879dcf7f631434fda
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Apr 23 19:51:09 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Apr 23 19:53:47 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=508c7667

portdbapi.async_fetch_map: pass loop to async_aux_get

Fixes: 8002d9343c38 ("portdbapi: add async_fetch_map method (bug 653810)")

 pym/portage/dbapi/porttree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 3cd929963..8a5fc690c 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -783,7 +783,7 @@ class portdbapi(dbapi):
{'EAPI':eapi,'SRC_URI':myuris}, use=useflags))
 
aux_get_future = self.async_aux_get(
-   mypkg, ["EAPI", "SRC_URI"], mytree=mytree)
+   mypkg, ["EAPI", "SRC_URI"], mytree=mytree, loop=loop)
result.add_done_callback(lambda result:
aux_get_future.cancel() if result.cancelled() else None)
aux_get_future.add_done_callback(aux_get_done)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

2018-04-23 Thread Zac Medico

commit: 8002d9343c385075ece19a131b0f584ec255a5b5
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 22 19:57:09 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Apr 23 18:39:16 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8002d934

portdbapi: add async_fetch_map method (bug 653810)

Add a portdbapi.async_fetch_map method which is identical to the
existing portdbapi.getFetchMap method, but returns a future. This
will be used by EbuildFetcher in order to avoid event loop recursion.

Bug: https://bugs.gentoo.org/653810
Reviewed-by: Brian Dolbec  gentoo.org>

 pym/portage/dbapi/porttree.py | 75 +--
 1 file changed, 58 insertions(+), 17 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 951e5760a..3cd929963 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -728,25 +728,66 @@ class portdbapi(dbapi):
URIs.
@rtype: dict
"""
+   loop = self._event_loop
+   return loop.run_until_complete(
+   self.async_fetch_map(mypkg, useflags=useflags,
+   mytree=mytree, loop=loop))
 
-   try:
-   eapi, myuris = self.aux_get(mypkg,
-   ["EAPI", "SRC_URI"], mytree=mytree)
-   except KeyError:
-   # Convert this to an InvalidDependString exception 
since callers
-   # already handle it.
-   raise portage.exception.InvalidDependString(
-   "getFetchMap(): aux_get() error reading 
"+mypkg+"; aborting.")
+   def async_fetch_map(self, mypkg, useflags=None, mytree=None, loop=None):
+   """
+   Asynchronous form of getFetchMap.
 
-   if not eapi_is_supported(eapi):
-   # Convert this to an InvalidDependString exception
-   # since callers already handle it.
-   raise portage.exception.InvalidDependString(
-   "getFetchMap(): '%s' has unsupported EAPI: 
'%s'" % \
-   (mypkg, eapi))
-
-   return _parse_uri_map(mypkg, {'EAPI':eapi,'SRC_URI':myuris},
-   use=useflags)
+   @param mypkg: cpv for an ebuild
+   @type mypkg: String
+   @param useflags: a collection of enabled USE flags, for 
evaluation of
+   conditionals
+   @type useflags: set, or None to enable all conditionals
+   @param mytree: The canonical path of the tree in which the 
ebuild
+   is located, or None for automatic lookup
+   @type mypkg: String
+   @param loop: event loop (defaults to global event loop)
+   @type loop: EventLoop
+   @return: A future that results in a dict which maps each file 
name to
+   a set of alternative URIs.
+   @rtype: asyncio.Future (or compatible)
+   """
+   loop = loop or global_event_loop()
+   loop = getattr(loop, '_asyncio_wrapper', loop)
+   result = loop.create_future()
+
+   def aux_get_done(aux_get_future):
+   if result.cancelled():
+   return
+   if aux_get_future.exception() is not None:
+   if isinstance(future.exception(), 
PortageKeyError):
+   # Convert this to an 
InvalidDependString exception since
+   # callers already handle it.
+   
result.set_exception(portage.exception.InvalidDependString(
+   "getFetchMap(): aux_get() error 
reading "
+   + mypkg + "; aborting."))
+   else:
+   result.set_exception(future.exception())
+   return
+
+   eapi, myuris = aux_get_future.result()
+
+   if not eapi_is_supported(eapi):
+   # Convert this to an InvalidDependString 
exception
+   # since callers already handle it.
+   
result.set_exception(portage.exception.InvalidDependString(
+   "getFetchMap(): '%s' has unsupported 
EAPI: '%s'" % \
+   (mypkg, eapi)))
+   return
+
+   result.set_result(_parse_uri_map(mypkg,
+   {'EAPI':eapi,'SRC_URI':myuris}, use=useflags))
+
+   aux_get_future = self.async_aux_get(
+

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/

2018-04-23 Thread Zac Medico

commit: c23d093d0330a9318534a1c521fb00d6360fabc0
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Apr 23 18:20:25 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Apr 23 18:24:11 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c23d093d

tests: prove run_until_complete executes done callbacks

Prove that done callbacks are executed before run_until_complete
returns, which is how asyncio's default event loop behaves. This
behavior was implemented in portage's internal event loop in commit
25245d7eb86ed197b7d7cfead0dbe4ce8ad4bc5b.

Bug: https://bugs.gentoo.org/591760

 .../futures/asyncio/test_run_until_complete.py | 29 ++
 1 file changed, 29 insertions(+)

diff --git a/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py 
b/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py
new file mode 100644
index 0..fc8f198ca
--- /dev/null
+++ b/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py
@@ -0,0 +1,29 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.util.futures import asyncio
+from portage.util.futures.unix_events import DefaultEventLoopPolicy
+
+
+class RunUntilCompleteTestCase(TestCase):
+   def test_add_done_callback(self):
+   initial_policy = asyncio.get_event_loop_policy()
+   if not isinstance(initial_policy, DefaultEventLoopPolicy):
+   asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
+
+   try:
+   loop = asyncio.get_event_loop()
+   f1 = loop.create_future()
+   f2 = loop.create_future()
+   f1.add_done_callback(f2.set_result)
+   loop.call_soon(lambda: f1.set_result(None))
+   loop.run_until_complete(f1)
+   self.assertEqual(f1.done(), True)
+
+   # This proves that done callbacks of f1 are executed 
before
+   # loop.run_until_complete(f1) returns, which is how 
asyncio's
+   # default event loop behaves.
+   self.assertEqual(f2.done(), True)
+   finally:
+   asyncio.set_event_loop_policy(initial_policy)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-22 Thread Zac Medico

commit: 3eb46b05eb098507ea74108559bf77006e0f8142
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Apr 23 00:49:15 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Apr 23 00:49:15 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3eb46b05

FileDigester: use _async_waitpid (bug 591760)

When pid exit status is not yet available, use the Subprocess
_async_waitpid() method to avoid event loop recursion.

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/FileDigester.py | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/pym/portage/util/_async/FileDigester.py 
b/pym/portage/util/_async/FileDigester.py
index 881c69280..72f06759c 100644
--- a/pym/portage/util/_async/FileDigester.py
+++ b/pym/portage/util/_async/FileDigester.py
@@ -60,8 +60,11 @@ class FileDigester(ForkProcess):
def _digest_pipe_reader_exit(self, pipe_reader):
self._parse_digests(pipe_reader.getvalue())
self._digest_pipe_reader = None
-   self._unregister()
-   self.wait()
+   if self.pid is None:
+   self._unregister()
+   self._async_wait()
+   else:
+   self._async_waitpid()
 
def _unregister(self):
ForkProcess._unregister(self)

[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/, pym/_emerge/

2018-04-22 Thread Zac Medico

commit: db4dca876cdb004b12a944f5323a51bc4bbde770
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Apr 23 00:04:29 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Apr 23 00:06:35 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=db4dca87

EbuildBuild._start(): fix event loop recursion (bug 653844)

Use async_aux_get to fetch SRC_URI metadata (it's not cached in
self.pkg.metadata because some packages have an extremely large
SRC_URI value).

Bug: https://bugs.gentoo.org/653844

 pym/_emerge/EbuildBuild.py | 19 ++-
 pym/portage/package/ebuild/doebuild.py |  6 --
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py
index 9d4afd0ea..8ad8bcae9 100644
--- a/pym/_emerge/EbuildBuild.py
+++ b/pym/_emerge/EbuildBuild.py
@@ -35,23 +35,32 @@ class EbuildBuild(CompositeTask):
("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", 
"_tree")
 
def _start(self):
-
-   pkg = self.pkg
-   settings = self.settings
-
if not self.opts.fetchonly:
-   rval = _check_temp_dir(settings)
+   rval = _check_temp_dir(self.settings)
if rval != os.EX_OK:
self.returncode = rval
self._current_task = None
self._async_wait()
return
 
+   # First get the SRC_URI metadata (it's not cached in 
self.pkg.metadata
+   # because some packages have an extremely large SRC_URI value).
+   self._start_task(
+   AsyncTaskFuture(
+   
future=self.pkg.root_config.trees["porttree"].dbapi.\
+   async_aux_get(self.pkg.cpv, ["SRC_URI"], 
myrepo=self.pkg.repo)),
+   self._start_with_metadata)
+
+   def _start_with_metadata(self, aux_get_task):
+   self._assert_current(aux_get_task)
+   pkg = self.pkg
+   settings = self.settings
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
+   settings.configdict["pkg"]["SRC_URI"], = 
aux_get_task.future.result()
settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
if self.opts.buildpkgonly:
settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly"

diff --git a/pym/portage/package/ebuild/doebuild.py 
b/pym/portage/package/ebuild/doebuild.py
index 3c8414387..0dabafeb7 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -450,8 +450,10 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
settings=None,
if hasattr(mydbapi, "getFetchMap") and \
("A" not in mysettings.configdict["pkg"] or \
"AA" not in mysettings.configdict["pkg"]):
-   src_uri, = mydbapi.aux_get(mysettings.mycpv,
-   ["SRC_URI"], mytree=mytree)
+   src_uri = mysettings.configdict["pkg"].get("SRC_URI")
+   if src_uri is None:
+   src_uri, = mydbapi.aux_get(mysettings.mycpv,
+   ["SRC_URI"], mytree=mytree)
metadata = {
"EAPI": eapi,
"SRC_URI" : src_uri,

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-22 Thread Zac Medico

commit: d836b0d32797f1431ebeaa20e2a18ea5cb7c3f7c
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 22 15:56:29 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Apr 22 17:48:52 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d836b0d3

AsyncFunction: use _async_waitpid (bug 591760)

When pid exit status is not yet available, use the Subprocess
_async_waitpid() method to avoid event loop recursion.

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/AsyncFunction.py | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/pym/portage/util/_async/AsyncFunction.py 
b/pym/portage/util/_async/AsyncFunction.py
index 40f6c5e75..ad3d8333f 100644
--- a/pym/portage/util/_async/AsyncFunction.py
+++ b/pym/portage/util/_async/AsyncFunction.py
@@ -57,8 +57,11 @@ class AsyncFunction(ForkProcess):
# and returned an unsuccessful returncode.
pass
self._async_func_reader = None
-   self._unregister()
-   self.wait()
+   if self.returncode is None:
+   self._async_waitpid()
+   else:
+   self._unregister()
+   self._async_wait()
 
def _unregister(self):
ForkProcess._unregister(self)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, pym/portage/tests/util/futures/

2018-04-22 Thread Zac Medico

commit: 6529d0e0ec017c4fa3048235649d25e71c967941
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 22 17:27:32 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Apr 22 17:27:32 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6529d0e0

test_iter_completed: fix max_load in testAsyncCancel

Since async_iter_completed sets a default multiprocessing.cpu_count()
value when max_load is None, use max_load=True to disable the load
limit.

Fixes: a9e8ebaa6979 ("Add async_iter_completed for asyncio migration (bug 
591760)")

 pym/portage/tests/util/futures/test_iter_completed.py | 2 +-
 pym/portage/util/_async/AsyncScheduler.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/tests/util/futures/test_iter_completed.py 
b/pym/portage/tests/util/futures/test_iter_completed.py
index b07146ed3..71343c22d 100644
--- a/pym/portage/tests/util/futures/test_iter_completed.py
+++ b/pym/portage/tests/util/futures/test_iter_completed.py
@@ -69,7 +69,7 @@ class IterCompletedTestCase(TestCase):
yield future
 
for future_done_set in async_iter_completed(future_generator(),
-   max_jobs=True, max_load=None, loop=loop):
+   max_jobs=True, max_load=True, loop=loop):
future_done_set.cancel()
break
 

diff --git a/pym/portage/util/_async/AsyncScheduler.py 
b/pym/portage/util/_async/AsyncScheduler.py
index 3deb6cb04..9beb8a848 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -14,7 +14,7 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
if max_jobs is None:
max_jobs = 1
self._max_jobs = max_jobs
-   self._max_load = max_load
+   self._max_load = None if max_load is True else max_load
self._error_count = 0
self._running_tasks = set()
self._remaining_tasks = True

[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/

2018-04-22 Thread Zac Medico

commit: a6e9c7cf429741015e26b923c8036416cc6bff7d
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 22 16:19:27 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Apr 22 16:24:37 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a6e9c7cf

test_iter_completed: fix SleepProcess._future_done cancel race

Fixes: a9e8ebaa6979 ("Add async_iter_completed for asyncio migration (bug 
591760)")

 pym/portage/tests/util/futures/test_iter_completed.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/tests/util/futures/test_iter_completed.py 
b/pym/portage/tests/util/futures/test_iter_completed.py
index 1344523c6..b07146ed3 100644
--- a/pym/portage/tests/util/futures/test_iter_completed.py
+++ b/pym/portage/tests/util/futures/test_iter_completed.py
@@ -19,7 +19,8 @@ class SleepProcess(ForkProcess):
ForkProcess._start(self)
 
def _future_done(self, task):
-   self.future.set_result(self.seconds)
+   if not self.future.cancelled():
+   self.future.set_result(self.seconds)
 
def _run(self):
time.sleep(self.seconds)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/

2018-04-19 Thread Zac Medico

commit: da5efb5e677d12c850ff02140e24a095a8efdafb
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Apr 19 06:36:32 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Apr 19 06:36:32 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=da5efb5e

EventLoop.run_until_complete: remove unneeded partial

Fixes: 25245d7eb86e ("EventLoop.run_until_complete: wait for done callbacks")

 pym/portage/util/_eventloop/EventLoop.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/util/_eventloop/EventLoop.py 
b/pym/portage/util/_eventloop/EventLoop.py
index 33fae26f2..6a8b906ed 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -818,7 +818,7 @@ class EventLoop(object):
# is easily achieved by registering a done callback and waiting 
for
# it to execute.
waiter = self.create_future()
-   future.add_done_callback(functools.partial(waiter.set_result))
+   future.add_done_callback(waiter.set_result)
while not waiter.done():
self.iteration()

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

2018-04-19 Thread Zac Medico

commit: 128c13ff8bd9a1685fb844f7a83469b2271f5e0f
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Apr 19 06:19:00 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Apr 19 06:25:32 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=128c13ff

AsyncTaskFuture: override _wait method

If the _wait method is called before the future is done, then wait for
it to complete.

 pym/portage/util/_async/AsyncTaskFuture.py | 5 +
 1 file changed, 5 insertions(+)

diff --git a/pym/portage/util/_async/AsyncTaskFuture.py 
b/pym/portage/util/_async/AsyncTaskFuture.py
index ee39183fe..c2316deeb 100644
--- a/pym/portage/util/_async/AsyncTaskFuture.py
+++ b/pym/portage/util/_async/AsyncTaskFuture.py
@@ -29,3 +29,8 @@ class AsyncTaskFuture(AsynchronousTask):
else:
self.returncode = 1
self.wait()
+
+   def _wait(self):
+   if self.returncode is None:
+   self.scheduler.run_until_complete(self.future)
+   return self.returncode

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/

2018-04-19 Thread Zac Medico

commit: 25245d7eb86ed197b7d7cfead0dbe4ce8ad4bc5b
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Apr 19 06:13:33 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Apr 19 06:13:33 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=25245d7e

EventLoop.run_until_complete: wait for done callbacks

Since done callbacks are executed via call_soon, it's desirable to
continue iterating until those callbacks have executed, which is easily
achieved by registering a done callback and waiting for it to execute.

 pym/portage/util/_eventloop/EventLoop.py | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/pym/portage/util/_eventloop/EventLoop.py 
b/pym/portage/util/_eventloop/EventLoop.py
index 7208c3aa1..33fae26f2 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -812,7 +812,14 @@ class EventLoop(object):
@raise: the Future's exception
"""
future = asyncio.ensure_future(future, 
loop=self._asyncio_wrapper)
-   while not future.done():
+
+   # Since done callbacks are executed via call_soon, it's 
desirable
+   # to continue iterating until those callbacks have executed, 
which
+   # is easily achieved by registering a done callback and waiting 
for
+   # it to execute.
+   waiter = self.create_future()
+   future.add_done_callback(functools.partial(waiter.set_result))
+   while not waiter.done():
self.iteration()
 
return future.result()

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/, pym/portage/package/ebuild/

2018-04-18 Thread Zac Medico

commit: cab78dba98c4309504e077c52a2ab0f0b7e27cc8
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Apr 17 20:56:49 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Apr 18 21:55:58 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cab78dba

emerge --usepkgonly: propagate implicit IUSE and USE_EXPAND (bug 640318)

For emerge --usepkgonly mode, it's useful to be able to operate without
dependence on a local ebuild repository and profile. Therefore,
merge Packages header settings from the binary package database
(including binhost(s) if enabled) into the profile configuration,
in order to propagate implicit IUSE and USE_EXPAND settings for use
with binary and installed packages. Values are appended, so the result
is a union of elements.

Also use ARCH from the binary package database if it's not defined by
the profile, since ARCH is used for validation by emerge's profile_check
function, and also for KEYWORDS logic in the _getmaskingstatus function.

All changes are currently confined to --usepkgonly mode, since this is
the mode where it is needed the most, and this ensures that behavior of
source-based builds is completely unaffected.

The changes only affect dependency calculations (where implicit IUSE
plays a role) and the user interface (display of USE_EXPAND flags).
The bash execution environment for binary and installed packages is
completely unaffected, since that environment relies on variables loaded
from environment.bz2 files.

Bug: https://bugs.gentoo.org/640318

 pym/_emerge/actions.py | 48 +
 pym/_emerge/main.py|  4 ---
 pym/portage/dbapi/bintree.py   | 56 ++
 pym/portage/package/ebuild/config.py   |  7 ++--
 pym/portage/package/ebuild/profile_iuse.py | 32 +
 5 files changed, 127 insertions(+), 20 deletions(-)

diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 347ef5fbc..b90aa8cb0 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -78,6 +78,7 @@ from _emerge.depgraph import backtrack_depgraph, depgraph, 
resume_depgraph
 from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange
 from _emerge.emergelog import emergelog
 from _emerge.is_valid_package_atom import is_valid_package_atom
+from _emerge.main import profile_check
 from _emerge.MetadataRegen import MetadataRegen
 from _emerge.Package import Package
 from _emerge.ProgressHandler import ProgressHandler
@@ -2209,9 +2210,21 @@ def action_uninstall(settings, trees, ldpath_mtimes,
return rval
 
 def adjust_configs(myopts, trees):
-   for myroot in trees:
+   for myroot, mytrees in trees.items():
mysettings =  trees[myroot]["vartree"].settings
mysettings.unlock()
+
+   # For --usepkgonly mode, propagate settings from the binary 
package
+   # database, so that it's possible to operate without dependence 
on
+   # a local ebuild repository and profile.
+   if ('--usepkgonly' in myopts and
+   mytrees['bintree']._propagate_config(mysettings)):
+   # Also propagate changes to the portdbapi 
doebuild_settings
+   # attribute which is used by Package instances for USE
+   # calculations (in support of --binpkg-respect-use).
+   mytrees['porttree'].dbapi.doebuild_settings = \
+   portage.config(clone=mysettings)
+
adjust_config(myopts, mysettings)
mysettings.lock()
 
@@ -2868,7 +2881,27 @@ def run_action(emerge_config):
"--usepkg", "--usepkgonly"):
emerge_config.opts.pop(opt, None)
 
+   # Populate the bintree with current --getbinpkg setting.
+   # This needs to happen before:
+   # * expand_set_arguments, in case any sets use the bintree
+   # * adjust_configs and profile_check, in order to propagate settings
+   #   implicit IUSE and USE_EXPAND settings from the binhost(s)
+   if (emerge_config.action in ('search', None) and
+   '--usepkg' in emerge_config.opts):
+   for mytrees in emerge_config.trees.values():
+   try:
+   mytrees['bintree'].populate(
+   getbinpkgs='--getbinpkg' in 
emerge_config.opts)
+   except ParseError as e:
+   writemsg('\n\n!!!%s.\nSee make.conf(5) for more 
info.\n'
+% (e,), noiselevel=-1)
+   return 1
+
adjust_configs(emerge_config.opts, emerge_config.trees)
+
+   if profile_check(emerge_config.trees, emerge_config.action) != os.EX_OK:
+   return 1
+
apply_priorities(emerge_config.target_config.settings)
 
if

1 2 3 4 5 6 7 >

1 - 100 of 623 matches

Mail list logo