[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: a7c7af98d755f34e84d1f0f847e2c0d5cc5b7e2f Author: Zac Medico gentoo org> AuthorDate: Wed Jul 18 03:36:59 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed Jul 18 03:40:05 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7c7af98 EventLoop: raise TypeError for unexpected call_* keyword args pym/portage/util/_eventloop/EventLoop.py | 24 1 file changed, 24 insertions(+) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 084ff0c18..ffd12cff9 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -859,11 +859,23 @@ class EventLoop(object): @return: a handle which can be used to cancel the callback @rtype: asyncio.Handle (or compatible) """ + try: + unexpected = next(key for key in kwargs if key != 'context') + except StopIteration: + pass + else: + raise TypeError("call_soon() got an unexpected keyword argument '%s'" % unexpected) return self._handle(self._idle_add( self._call_soon_callback(callback, args)), self) def call_soon_threadsafe(self, callback, *args, **kwargs): """Like call_soon(), but thread safe.""" + try: + unexpected = next(key for key in kwargs if key != 'context') + except StopIteration: + pass + else: + raise TypeError("call_soon_threadsafe() got an unexpected keyword argument '%s'" % unexpected) # idle_add provides thread safety return self._handle(self.idle_add( self._call_soon_callback(callback, args)), self) @@ -909,6 +921,12 @@ class EventLoop(object): @return: a handle which can be used to cancel the callback @rtype: asyncio.Handle (or compatible) """ + try: + unexpected = next(key for key in kwargs if key != 'context') + except StopIteration: + pass + else: + raise TypeError("call_later() got an unexpected keyword argument '%s'" % unexpected) return self._handle(self.timeout_add( delay * 1000, self._call_soon_callback(callback, args)), self) @@ -936,6 +954,12 @@ class EventLoop(object): @return: a handle which can be used to cancel the callback @rtype: asyncio.Handle (or compatible) """ + try: + unexpected = next(key for key in kwargs if key != 'context') + except StopIteration: + pass + else: + raise TypeError("call_at() got an unexpected keyword argument '%s'" % unexpected) delta = when - self.time() return self.call_later(delta if delta > 0 else 0, callback, *args)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: e46dd735cd4dde58cf3f8ef3cd2b8b29561f5b3e Author: Zac Medico gentoo org> AuthorDate: Tue Jul 17 19:27:28 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue Jul 17 19:27:28 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e46dd735 EventLoop: use python2.7 compatible **kwargs for call_* context arg Since python2.7 does not allow positional default arguments after *args, use **kwargs instead. Fixes: ae8cc32ccd81 ("EventLoop: add call_* context arg for python3.7 compat") pym/portage/util/_eventloop/EventLoop.py | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 69ccbac2c..084ff0c18 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -832,7 +832,7 @@ class EventLoop(object): return future.result() - def call_soon(self, callback, *args, context=None): + def call_soon(self, callback, *args, **kwargs): """ Arrange for a callback to be called as soon as possible. The callback is called after call_soon() returns, when control returns to the event @@ -862,7 +862,7 @@ class EventLoop(object): return self._handle(self._idle_add( self._call_soon_callback(callback, args)), self) - def call_soon_threadsafe(self, callback, *args, context=None): + def call_soon_threadsafe(self, callback, *args, **kwargs): """Like call_soon(), but thread safe.""" # idle_add provides thread safety return self._handle(self.idle_add( @@ -877,7 +877,7 @@ class EventLoop(object): """ return monotonic() - def call_later(self, delay, callback, *args, context=None): + def call_later(self, delay, callback, *args, **kwargs): """ Arrange for the callback to be called after the given delay seconds (either an int or float). @@ -912,7 +912,7 @@ class EventLoop(object): return self._handle(self.timeout_add( delay * 1000, self._call_soon_callback(callback, args)), self) - def call_at(self, when, callback, *args, context=None): + def call_at(self, when, callback, *args, **kwargs): """ Arrange for the callback to be called at the given absolute timestamp when (an int or float), using the same time reference as
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: ae8cc32ccd812661650647feffa1b10fc3ab5837 Author: Zac Medico gentoo org> AuthorDate: Tue Jul 17 19:04:28 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue Jul 17 19:05:38 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ae8cc32c EventLoop: add call_* context arg for python3.7 compat The context argument currently does nothing, but exists for minimal interoperability with Future instances that require it for PEP 567. pym/portage/util/_eventloop/EventLoop.py | 26 ++ 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index df76374d9..69ccbac2c 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -832,7 +832,7 @@ class EventLoop(object): return future.result() - def call_soon(self, callback, *args): + def call_soon(self, callback, *args, context=None): """ Arrange for a callback to be called as soon as possible. The callback is called after call_soon() returns, when control returns to the event @@ -844,18 +844,25 @@ class EventLoop(object): Any positional arguments after the callback will be passed to the callback when it is called. + The context argument currently does nothing, but exists for minimal + interoperability with Future instances that require it for PEP 567. + An object compatible with asyncio.Handle is returned, which can be used to cancel the callback. @type callback: callable @param callback: a function to call + @type context: contextvars.Context + @param context: An optional keyword-only context argument allows + specifying a custom contextvars.Context for the callback to run + in. The current context is used when no context is provided. @return: a handle which can be used to cancel the callback @rtype: asyncio.Handle (or compatible) """ return self._handle(self._idle_add( self._call_soon_callback(callback, args)), self) - def call_soon_threadsafe(self, callback, *args): + def call_soon_threadsafe(self, callback, *args, context=None): """Like call_soon(), but thread safe.""" # idle_add provides thread safety return self._handle(self.idle_add( @@ -870,7 +877,7 @@ class EventLoop(object): """ return monotonic() - def call_later(self, delay, callback, *args): + def call_later(self, delay, callback, *args, context=None): """ Arrange for the callback to be called after the given delay seconds (either an int or float). @@ -886,19 +893,26 @@ class EventLoop(object): it is called. If you want the callback to be called with some named arguments, use a closure or functools.partial(). + The context argument currently does nothing, but exists for minimal + interoperability with Future instances that require it for PEP 567. + Use functools.partial to pass keywords to the callback. @type delay: int or float @param delay: delay seconds @type callback: callable @param callback: a function to call + @type context: contextvars.Context + @param context: An optional keyword-only context argument allows + specifying a custom contextvars.Context for the callback to run + in. The current context is used when no context is provided. @return: a handle which can be used to cancel the callback @rtype: asyncio.Handle (or compatible) """ return self._handle(self.timeout_add( delay * 1000, self._call_soon_callback(callback, args)), self) - def call_at(self, when, callback, *args): + def call_at(self, when, callback, *args, context=None): """ Arrange for the callback to be called at the given absolute timestamp when (an int or float), using the same time reference as @@ -915,6 +929,10 @@ class EventLoop(object): @param when: absolute timestamp when to call callback @type callback: callable @param callback: a function to call + @type context: contextvars.Context + @param context: An optional keyword-only context argument allows + specifying a custom contextvars.Context for the callback to run +
[gentoo-commits] proj/portage:master commit in: pym/portage/repository/
commit: 01f49cb12c66026e5b225912f9759b50ff2e2289 Author: Zac Medico gentoo org> AuthorDate: Sun Jul 15 20:55:48 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Jul 16 05:53:05 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=01f49cb1 RepoConfig: fix make.conf PORTDIR override (bug 661276) Pass the main-repo name from repos.conf as the repository name for the PORTDIR RepoConfig constructor, so that it can override the main repo location even if the profiles/repo_name file does not exist yet (like in a stage3 tarball). Bug: https://bugs.gentoo.org/661276 Reported-by: Jorge Manuel B. S. Vicetto gentoo.org> pym/portage/repository/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index bf2b6dd03..e45e67dec 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -524,7 +524,8 @@ class RepoConfigLoader(object): (base_priority == 0 and ov is portdir): repo_opts = default_repo_opts.copy() repo_opts['location'] = ov - repo = RepoConfig(None, repo_opts, local_config=local_config) + name = prepos['DEFAULT'].main_repo if ov is portdir else None + repo = RepoConfig(name, repo_opts, local_config=local_config) # repos_conf_opts contains options from repos.conf repos_conf_opts = repos_conf.get(repo.name) if repos_conf_opts is not None:
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
commit: 84431fe188ed1b3e2e7b52918329bd696d8304b1 Author: Zac Medico gentoo org> AuthorDate: Mon Jul 16 05:07:10 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Jul 16 05:18:51 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=84431fe1 portdbapi.cp_list: honor porttrees modifications (bug 650814) Consumers, such as repoman and emirrordist, may modify the porttrees attribute in order to modify the effective set of repositories for all portdbapi operations. Fixes: 27eeeb6e4fc8 ("portdbapi.cp_list: cache repo associations (bug 650814)") Bug: https://bugs.gentoo.org/650814 pym/portage/dbapi/porttree.py | 25 +++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 2e271ea76..677452273 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -322,6 +322,26 @@ class portdbapi(dbapi): self._better_cache = None self._broken_ebuilds = set() + def _set_porttrees(self, porttrees): + """ + Consumers, such as repoman and emirrordist, may modify the porttrees + attribute in order to modify the effective set of repositories for + all portdbapi operations. + + @param porttrees: list of repo locations, in ascending order by + repo priority + @type porttrees: list + """ + self._porttrees_repos = portage.OrderedDict((repo.name, repo) + for repo in (self.repositories.get_repo_for_location(location) + for location in porttrees)) + self._porttrees = tuple(porttrees) + + def _get_porttrees(self): + return self._porttrees + + porttrees = property(_get_porttrees, _set_porttrees) + @property def _event_loop(self): if portage._internal_caller: @@ -972,9 +992,10 @@ class portdbapi(dbapi): repos = [self.repositories.get_repo_for_location(location) for location in mytree] elif self._better_cache is None: - repos = list(self.repositories) + repos = self._porttrees_repos.values() else: - repos = reversed(self._better_cache[mycp]) + repos = [repo for repo in reversed(self._better_cache[mycp]) + if repo.name in self._porttrees_repos] mylist = [] for repo in repos: oroot = repo.location
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/
commit: 2de6ba26fd45fe942329ac928029e4edf603e125 Author: Zac Medico gentoo org> AuthorDate: Sun Jul 15 04:54:45 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Jul 15 21:47:33 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2de6ba26 config: cache profile.bashrc stat results (bug 649806) Optimize config to stat profile.bashrc files once in the constructor, in order to avoid repeated stat calls in the setcpv method. Bug: https://bugs.gentoo.org/649806 Reviewed-by: Brian Dolbec gentoo.org> pym/portage/package/ebuild/config.py | 13 - 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index 88acac5cc..320d9f6c0 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -274,6 +274,7 @@ class config(object): self.mycpv = clone.mycpv self._setcpv_args_hash = clone._setcpv_args_hash self._soname_provided = clone._soname_provided + self._profile_bashrc = clone._profile_bashrc # immutable attributes (internal policy ensures lack of mutation) self._locations_manager = clone._locations_manager @@ -725,6 +726,10 @@ class config(object): self._license_manager.extract_global_changes( \ self.configdict["conf"].get("ACCEPT_LICENSE", "")) + # profile.bashrc + self._profile_bashrc = tuple(os.path.isfile(os.path.join(profile.location, 'profile.bashrc')) + for profile in profiles_complex) + if local_config: #package.properties propdict = grabdict_package(os.path.join( @@ -1596,11 +1601,9 @@ class config(object): bashrc_files = [] - for profile in self._locations_manager.profiles_complex: - profile_bashrc = os.path.join(profile.location, - 'profile.bashrc') - if os.path.exists(profile_bashrc): - bashrc_files.append(profile_bashrc) + for profile, profile_bashrc in zip(self._locations_manager.profiles_complex, self._profile_bashrc): + if profile_bashrc: + bashrc_files.append(os.path.join(profile.location, 'profile.bashrc')) if profile in self._pbashrcdict: cpdict = self._pbashrcdict[profile].get(cp) if cpdict:
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
commit: 31edfe62541d8b55c64584b056ce64ad759f6a8e Author: Zac Medico gentoo org> AuthorDate: Sun Jul 15 21:38:30 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Jul 15 21:38:30 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=31edfe62 portdbapi.xmatch: return separate items for each repo Since xmatch match-all shares cache with the cp_list method, it needs to return separate items when the same package version is found in multiple repos. Fixes: 27eeeb6e4fc8 ("portdbapi.cp_list: cache repo associations (bug 650814)") Bug: https://bugs.gentoo.org/650814 pym/portage/dbapi/porttree.py | 19 +++ 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index f6076ee2b..2e271ea76 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -1108,20 +1108,10 @@ class portdbapi(dbapi): else: iterfunc = iter - if mydep.repo is not None: - repos = [mydep.repo] - else: - # We iterate over self.porttrees, since it's common to - # tweak this attribute in order to adjust match behavior. - repos = [] - for tree in reversed(self.porttrees): - repos.append(self.repositories.get_name_for_location(tree)) - for cpv in iterfunc(mylist): - for repo in repos: try: metadata = dict(zip(aux_keys, - self.aux_get(cpv, aux_keys, myrepo=repo))) + self.aux_get(cpv, aux_keys, myrepo=cpv.repo))) except KeyError: # ebuild not in this repo, or masked by corruption continue @@ -1145,11 +1135,8 @@ class portdbapi(dbapi): continue myval.append(pkg_str) - # only yield a given cpv once - break - - if myval and single_match: - break + if single_match: + break if single_match: if myval:
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
commit: 3a25c3fa13d7c62ba8c00d6c7a75eb907d34b568 Author: Zac Medico gentoo org> AuthorDate: Fri Jul 13 03:58:04 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Jul 14 23:54:29 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3a25c3fa _unmerge_dirs: revisit parents of removed symlinks (bug 640058) When removal of a symlink is triggered by removal of the directory that it points to, revisit the parent directories of the symlink. Bug: https://bugs.gentoo.org/640058 Reviewed-by: Brian Dolbec gentoo.org> pym/portage/dbapi/vartree.py | 25 +++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 1a86940f1..a104306eb 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -2753,9 +2753,13 @@ class dblink(object): real_root = self.settings['ROOT'] dirs = sorted(dirs) - dirs.reverse() + revisit = {} - for obj, inode_key in dirs: + while True: + try: + obj, inode_key = dirs.pop() + except IndexError: + break # Treat any directory named "info" as a candidate here, # since it might have been in INFOPATH previously even # though it may not be there now. @@ -2818,6 +2822,7 @@ class dblink(object): raise if e.errno != errno.ENOENT: show_unmerge("---", unmerge_desc["!empty"], "dir", obj) + revisit[obj] = inode_key # Since we didn't remove this directory, record the directory # itself for use in syncfs calls, if we have removed another @@ -2838,6 +2843,7 @@ class dblink(object): # no need to protect symlinks that point to it. unmerge_syms = protected_symlinks.pop(inode_key, None) if unmerge_syms is not None: + parents = [] for relative_path in unmerge_syms: obj = os.path.join(real_root, relative_path.lstrip(os.sep)) @@ -2849,6 +2855,21 @@ class dblink(object): raise del e show_unmerge("!!!", "", "sym", obj) + else: + parents.append(os.path.dirname(obj)) + + if parents: + # Revisit parents recursively (bug 640058). + recursive_parents = [] + for parent in set(parents): + while parent in revisit: + recursive_parents.append(parent) + parent = os.path.dirname(parent) + if parent == '/': + break + + for parent in sorted(set(recursive_parents)): + dirs.append((parent, revisit.pop(parent))) def isowner(self, filename, destroot=None): """
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/, pym/portage/util/futures/
commit: edba848a013e1797faeacc1911a5e6571fca3ca7 Author: Zac Medico gentoo org> AuthorDate: Thu Jul 5 04:44:42 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed Jul 11 07:40:59 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=edba848a Add python2 compatible coroutine support (bug 660426) For readability, it's desirable to make asynchronous code use coroutines to avoid callbacks when possible. For python2 compatibility, generators that yield Futures can be used to implement coroutines. Add a compat_coroutine module which provides a @coroutine decorator and a coroutine_return function that can be used to return a value from a generator. The decorated function returns a Future which is done when the generator is exhausted. Usage is very similar to asyncio coroutine usage in python3.4 (see unit tests). Bug: https://bugs.gentoo.org/660426 .../tests/util/futures/test_compat_coroutine.py| 159 + pym/portage/util/futures/compat_coroutine.py | 112 +++ 2 files changed, 271 insertions(+) diff --git a/pym/portage/tests/util/futures/test_compat_coroutine.py b/pym/portage/tests/util/futures/test_compat_coroutine.py new file mode 100644 index 0..cbc070869 --- /dev/null +++ b/pym/portage/tests/util/futures/test_compat_coroutine.py @@ -0,0 +1,159 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import ( + coroutine, + coroutine_return, +) +from portage.tests import TestCase + + +class CompatCoroutineTestCase(TestCase): + + def test_returning_coroutine(self): + @coroutine + def returning_coroutine(): + yield asyncio.sleep(0) + coroutine_return('success') + + self.assertEqual('success', + asyncio.get_event_loop().run_until_complete(returning_coroutine())) + + def test_raising_coroutine(self): + + class TestException(Exception): + pass + + @coroutine + def raising_coroutine(): + yield asyncio.sleep(0) + raise TestException('exception') + + self.assertRaises(TestException, + asyncio.get_event_loop().run_until_complete, raising_coroutine()) + + def test_catching_coroutine(self): + + class TestException(Exception): + pass + + @coroutine + def catching_coroutine(loop=None): + loop = asyncio._wrap_loop(loop) + future = loop.create_future() + loop.call_soon(future.set_exception, TestException('exception')) + try: + yield future + except TestException: + self.assertTrue(True) + else: + self.assertTrue(False) + coroutine_return('success') + + loop = asyncio.get_event_loop() + self.assertEqual('success', + loop.run_until_complete(catching_coroutine(loop=loop))) + + def test_cancelled_coroutine(self): + + @coroutine + def cancelled_coroutine(loop=None): + loop = asyncio._wrap_loop(loop) + while True: + yield loop.create_future() + + loop = asyncio.get_event_loop() + future = cancelled_coroutine(loop=loop) + loop.call_soon(future.cancel) + + self.assertRaises(asyncio.CancelledError, + loop.run_until_complete, future) + + def test_cancelled_future(self): + + @coroutine + def cancelled_future_coroutine(loop=None): + loop = asyncio._wrap_loop(loop) + while True: + future = loop.create_future() + loop.call_soon(future.cancel) + yield future + + loop = asyncio.get_event_loop() + self.assertRaises(asyncio.CancelledError, + loop.run_until_complete, cancelled_future_coroutine(loop=loop)) + + def test_yield_expression_result(self): + @coroutine + def yield_expression_coroutine(): + for i in range(3): + x = yield asyncio.sleep(0, result=i) + self.assertEqual(x, i) + + asyncio.get_event_loop().run_until_complete(yield_expression_coroutine()) + + def test_method_coroutine(self): + + class Cubby(object): + +
[gentoo-commits] proj/portage:master commit in: pym/portage/repository/, man/, pym/portage/sync/modules/git/
commit: 903c4b1a67689c4b8cc59113a56d58575cf7db8e Author: Zac Medico gentoo org> AuthorDate: Tue Jul 10 07:03:35 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed Jul 11 06:10:41 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=903c4b1a GitSync: support sync-depth (bug 552814) Support sync-depth for shallow sync, using git reset --merge just like in the earlier implementation that was reverted in commit ab840ac982d3c8b676b89f6bedd14e85dd06870f. Also, run git gc --auto in the foreground, in order to trigger periodic housekeeping and hopefully avoid errors from automatic git gc calls as reported in bug 599008. The default sync-depth is unlimited, which means that default behavior remains unchanged (unlike the previous implementation that was reverted). Bug: https://bugs.gentoo.org/552814 Bug: https://bugs.gentoo.org/599008 man/portage.5 | 3 ++- pym/portage/repository/config.py| 4 pym/portage/sync/modules/git/git.py | 26 +- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/man/portage.5 b/man/portage.5 index acc80791b..a57531d44 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -985,7 +985,8 @@ overlay filesystems. Specifies CVS repository. .TP .B sync\-depth -This is a deprecated alias for the \fBclone\-depth\fR option. +Specifies sync depth to use for DVCS repositories. If set to 0, the +depth is unlimited. Defaults to 0. .TP .B sync\-git\-clone\-env Set environment variables for git when cloning repository (git clone). diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index ad7ae9d18..bf2b6dd03 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -179,10 +179,6 @@ class RepoConfig(object): self.clone_depth = repo_opts.get('clone-depth') self.sync_depth = repo_opts.get('sync-depth') - if self.sync_depth is not None: - warnings.warn(_("repos.conf: sync-depth is deprecated," - " use clone-depth instead")) - self.sync_hooks_only_on_change = repo_opts.get( 'sync-hooks-only-on-change', 'false').lower() == 'true' diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py index 68f8bd1fb..2fb82c600 100644 --- a/pym/portage/sync/modules/git/git.py +++ b/pym/portage/sync/modules/git/git.py @@ -137,6 +137,24 @@ class GitSync(NewBase): writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1) return (e.returncode, False) + shallow = self.repo.sync_depth is not None and self.repo.sync_depth != 0 + if shallow: + git_cmd_opts += " --depth %d" % self.repo.sync_depth + + # For shallow fetch, unreachable objects may need to be pruned + # manually, in order to prevent automatic git gc calls from + # eventually failing (see bug 599008). + gc_cmd = ['git', '-c', 'gc.autodetach=false', 'gc', '--auto'] + if quiet: + gc_cmd.append('--quiet') + exitcode = subprocess.call(gc_cmd, + cwd=portage._unicode_encode(self.repo.location)) + if exitcode != os.EX_OK: + msg = "!!! git gc error in %s" % self.repo.location + self.logger(self.xterm_titles, msg) + writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1) + return (exitcode, False) + git_cmd = "%s fetch %s%s" % (self.bin_command, remote_branch.partition('/')[0], git_cmd_opts) @@ -159,7 +177,13 @@ class GitSync(NewBase): if not self.verify_head(revision='refs/remotes/%s' % remote_branch): return (1, False) - merge_cmd = [self.bin_command, 'merge', 'refs/remotes/%s' % remote_branch] + if shallow: + # Since the default merge strategy typically fails when + # the depth is not unlimited, `git reset --merge`. + merge_cmd = [self.bin_command, 'reset', '--merge'] + else: + merge_cmd = [self.bin_command, 'merge'] + merge_cmd.append('refs/remotes/%s' % remote_branch) if quiet: merge_cmd.append('--quiet') exitcode = subprocess.call(merge_cmd,
[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/, cnf/, pym/portage/repository/, man/
commit: 84822ef7a21494d3f044c2ffa7b112e4d29665ab Author: Zac Medico gentoo org> AuthorDate: Thu Jul 5 13:10:43 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue Jul 10 05:03:53 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=84822ef7 rsync: quarantine data prior to verification (bug 660410) Sync into a quarantine subdirectory, using the rsync --link-dest option to create hardlinks to identical files in the previous snapshot of the repository. If hardlinks are not supported, then show a warning message and sync directly to the normal repository location. If verification succeeds, then the quarantine subdirectory is synced to the normal repository location, and the quarantine subdirectory is deleted. If verification fails, then the quarantine directory is preserved for purposes of analysis. Even if verification happens to be disabled, the quarantine directory is still useful for making the repository update more atomic, so that it is less likely that normal repository location will be observed in a partially synced state. The new behavior may conflict with configurations that restrict the use of hardlinks, such as overlay filesystems. Therefore, users will have to set "sync-allow-hardlinks = no" in repos.conf if they have a configuration that prevents the use of hardlinks, but this should not be very common. Bug: https://bugs.gentoo.org/660410 cnf/repos.conf | 1 + man/portage.5 | 8 +++ pym/portage/repository/config.py| 7 ++- pym/portage/sync/modules/rsync/rsync.py | 87 ++--- 4 files changed, 94 insertions(+), 9 deletions(-) diff --git a/cnf/repos.conf b/cnf/repos.conf index 352073cfd..419f6d118 100644 --- a/cnf/repos.conf +++ b/cnf/repos.conf @@ -6,6 +6,7 @@ location = /usr/portage sync-type = rsync sync-uri = rsync://rsync.gentoo.org/gentoo-portage auto-sync = yes +sync-allow-hardlinks = yes sync-rsync-verify-jobs = 1 sync-rsync-verify-metamanifest = yes sync-rsync-verify-max-age = 24 diff --git a/man/portage.5 b/man/portage.5 index 5adb07d82..acc80791b 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -973,6 +973,14 @@ files). Defaults to true. .br Valid values: true, false. .TP +.B sync\-allow\-hardlinks = yes|no +Allow sync plugins to use hardlinks in order to ensure that a repository +remains in a valid state if something goes wrong during the sync operation. +For example, if signature verification fails during a sync operation, +the previous state of the repository will be preserved. This option may +conflict with configurations that restrict the use of hardlinks, such as +overlay filesystems. +.TP .B sync\-cvs\-repo Specifies CVS repository. .TP diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index 1d897bb90..ad7ae9d18 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -86,6 +86,7 @@ class RepoConfig(object): 'sync_type', 'sync_umask', 'sync_uri', 'sync_user', 'thin_manifest', 'update_changelog', '_eapis_banned', '_eapis_deprecated', '_masters_orig', 'module_specific_options', 'manifest_required_hashes', + 'sync_allow_hardlinks', 'sync_openpgp_key_path', 'sync_openpgp_key_refresh_retry_count', 'sync_openpgp_key_refresh_retry_delay_max', @@ -188,6 +189,9 @@ class RepoConfig(object): self.strict_misc_digests = repo_opts.get( 'strict-misc-digests', 'true').lower() == 'true' + self.sync_allow_hardlinks = repo_opts.get( + 'sync-allow-hardlinks', 'true').lower() in ('true', 'yes') + self.sync_openpgp_key_path = repo_opts.get( 'sync-openpgp-key-path', None) @@ -534,6 +538,7 @@ class RepoConfigLoader(object): 'clone_depth', 'eclass_overrides', 'force', 'masters', 'priority', 'strict_misc_digests', 'sync_depth', 'sync_hooks_only_on_change', + 'sync_allow_hardlinks', 'sync_openpgp_key_path', 'sync_openpgp_key_refresh_retry_count', 'sync_openpgp_key_refresh_retry_delay_max', @@ -960,7 +965,7 @@ class RepoConfigLoader(object): return repo_name in self.prepos def config_string(self): - bool_keys = ("strict_misc_digests",) + bool_keys = ("strict_misc_digests", "sync_allow_hardlinks") str_or_int_keys = ("auto_sync", "clone_depth", "format", "location", "main_repo", "prio
[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/
commit: 89b85e47d7ac0d5f36b182c36eb1e72db7187b36 Author: Zac Medico gentoo org> AuthorDate: Sun Jul 8 20:29:40 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue Jul 10 04:25:54 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=89b85e47 GitSync: add key refresh retry (bug 660732) Bug: https://bugs.gentoo.org/660732 pym/portage/sync/modules/git/git.py | 7 +++ 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py index 97c4c1de6..68f8bd1fb 100644 --- a/pym/portage/sync/modules/git/git.py +++ b/pym/portage/sync/modules/git/git.py @@ -8,6 +8,7 @@ import subprocess import portage from portage import os from portage.util import writemsg_level, shlex_split +from portage.util.futures import asyncio from portage.output import create_color_func, EOutput good = create_color_func("GOOD") bad = create_color_func("BAD") @@ -197,10 +198,8 @@ class GitSync(NewBase): out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,)) with io.open(self.repo.sync_openpgp_key_path, 'rb') as f: openpgp_env.import_key(f) - out.ebegin('Refreshing keys from keyserver') - openpgp_env.refresh_keys() - out.eend(0) - except GematoException as e: + self._refresh_keys(openpgp_env) + except (GematoException, asyncio.TimeoutError) as e: writemsg_level("!!! Verification impossible due to keyring problem:\n%s\n" % (e,), level=logging.ERROR, noiselevel=-1)
[gentoo-commits] proj/portage:master commit in: pym/portage/sync/, pym/portage/sync/modules/rsync/
commit: 06f304c8718ac653bbdf9b5b999c03cba898bf3e Author: Zac Medico gentoo org> AuthorDate: Sun Jul 8 20:09:45 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue Jul 10 04:23:12 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=06f304c8 SyncBase: split out _refresh_keys method (bug 660732) Split out a _refresh_keys method from the RsyncSync class, so GitSync can use it for retry support. Bug: https://bugs.gentoo.org/660732 pym/portage/sync/modules/rsync/rsync.py | 33 +- pym/portage/sync/syncbase.py| 41 + 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py index 382a1eaae..a715e2818 100644 --- a/pym/portage/sync/modules/rsync/rsync.py +++ b/pym/portage/sync/modules/rsync/rsync.py @@ -7,7 +7,6 @@ import time import signal import socket import datetime -import functools import io import re import random @@ -23,10 +22,8 @@ good = create_color_func("GOOD") bad = create_color_func("BAD") warn = create_color_func("WARN") from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM -from portage.util._eventloop.global_event_loop import global_event_loop from portage.util import writemsg, writemsg_stdout from portage.util.futures import asyncio -from portage.util.futures.executor.fork import ForkExecutor from portage.sync.getaddrinfo_validate import getaddrinfo_validate from _emerge.UserQuery import UserQuery from portage.sync.syncbase import NewBase @@ -149,39 +146,11 @@ class RsyncSync(NewBase): # will not be performed and the user will have to fix it and try again, # so we may as well bail out before actual rsync happens. if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None: - try: out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,)) with io.open(self.repo.sync_openpgp_key_path, 'rb') as f: openpgp_env.import_key(f) - out.ebegin('Refreshing keys from keyserver') - retry_decorator = self._key_refresh_retry_decorator() - if retry_decorator is None: - openpgp_env.refresh_keys() - else: - def noisy_refresh_keys(): - """ - Since retry does not help for some types of - errors, display errors as soon as they occur. - """ - try: - openpgp_env.refresh_keys() - except Exception as e: - writemsg_level("%s\n" % (e,), - level=logging.ERROR, noiselevel=-1) - raise # retry - - # The ThreadPoolExecutor that asyncio uses by default - # does not support cancellation of tasks, therefore - # use ForkExecutor for task cancellation support, in - # order to enforce timeouts. - loop = global_event_loop() - with ForkExecutor(loop=loop) as executor: - func_coroutine = functools.partial(loop.run_in_executor, - executor, noisy_refresh_keys) - decorated_func = retry_decorator(func_coroutine, loop=loop) - loop.run_until_complete(decorated_func()) - out.eend(0) + self._refresh_keys(openpgp_env) except (GematoException, asyncio.TimeoutError) as e: writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n" % (e,), diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py index
[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/
commit: dac12abb20592b098163c87572f6e04416ef3996 Author: Zac Medico gentoo org> AuthorDate: Sun Jul 8 22:15:09 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Jul 8 22:15:09 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dac12abb GitSync: fix verification of remote branch Fixes: 3cd8cf93abb6 ("GitSync: abort checkout for signature problem (bug 660372)") pym/portage/sync/modules/git/git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py index 85a44289a..97c4c1de6 100644 --- a/pym/portage/sync/modules/git/git.py +++ b/pym/portage/sync/modules/git/git.py @@ -155,7 +155,7 @@ class GitSync(NewBase): writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1) return (exitcode, False) - if not self.verify_head(revision='refs/remotes/%s^..' % remote_branch): + if not self.verify_head(revision='refs/remotes/%s' % remote_branch): return (1, False) merge_cmd = [self.bin_command, 'merge', 'refs/remotes/%s' % remote_branch] @@ -209,7 +209,7 @@ class GitSync(NewBase): env = os.environ.copy() env['GNUPGHOME'] = openpgp_env.home - rev_cmd = [self.bin_command, "log", "--pretty=format:%G?", revision] + rev_cmd = [self.bin_command, "log", "-n1", "--pretty=format:%G?", revision] try: status = (portage._unicode_decode( subprocess.check_output(rev_cmd,
[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/
commit: 3cd8cf93abb6410cc877381531bb662a704dffa7 Author: Zac Medico gentoo org> AuthorDate: Thu Jul 5 10:10:36 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Jul 8 21:16:31 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3cd8cf93 GitSync: abort checkout for signature problem (bug 660372) Fetch the upstream remote and use git merge to update the checkout only after successful verification of the upstream head. Suggested-by: Richard Freeman gentoo.org> Reviewed-by: Arfrever Frehtes Taifersar Arahesis Apache.Org> Bug: https://bugs.gentoo.org/660372 pym/portage/sync/modules/git/git.py | 39 - 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py index 160137a6d..85a44289a 100644 --- a/pym/portage/sync/modules/git/git.py +++ b/pym/portage/sync/modules/git/git.py @@ -109,6 +109,7 @@ class GitSync(NewBase): if not self.has_bin: return (1, False) git_cmd_opts = "" + quiet = self.settings.get("PORTAGE_QUIET") == "1" if self.repo.module_specific_options.get('sync-git-env'): shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env']) env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k) @@ -123,7 +124,21 @@ class GitSync(NewBase): git_cmd_opts += " --quiet" if self.repo.module_specific_options.get('sync-git-pull-extra-opts'): git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-pull-extra-opts'] - git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts) + + try: + remote_branch = portage._unicode_decode( + subprocess.check_output([self.bin_command, 'rev-parse', + '--abbrev-ref', '--symbolic-full-name', '@{upstream}'], + cwd=portage._unicode_encode(self.repo.location))).rstrip('\n') + except subprocess.CalledProcessError as e: + msg = "!!! git rev-parse error in %s" % self.repo.location + self.logger(self.xterm_titles, msg) + writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1) + return (e.returncode, False) + + git_cmd = "%s fetch %s%s" % (self.bin_command, + remote_branch.partition('/')[0], git_cmd_opts) + writemsg_level(git_cmd + "\n") rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"] @@ -133,20 +148,34 @@ class GitSync(NewBase): exitcode = portage.process.spawn_bash("cd %s ; exec %s" % ( portage._shell_quote(self.repo.location), git_cmd), **self.spawn_kwargs) + if exitcode != os.EX_OK: - msg = "!!! git pull error in %s" % self.repo.location + msg = "!!! git fetch error in %s" % self.repo.location self.logger(self.xterm_titles, msg) writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1) return (exitcode, False) - if not self.verify_head(): + + if not self.verify_head(revision='refs/remotes/%s^..' % remote_branch): return (1, False) + merge_cmd = [self.bin_command, 'merge', 'refs/remotes/%s' % remote_branch] + if quiet: + merge_cmd.append('--quiet') + exitcode = subprocess.call(merge_cmd, + cwd=portage._unicode_encode(self.repo.location)) + + if exitcode != os.EX_OK: + msg = "!!! git merge error in %s" % self.repo.location + self.logger(self.xterm_titles, msg) + writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1) + return (exitcode, False) + current_rev = subprocess.check_output(rev_cmd, cwd=portage._unicode_encode(self.repo.location)) return (os.EX_OK, current_rev != previous_rev) - def verify_head(self): + def verify_head(self, revision='-1'): if (self.repo.module_specific_options.get( 'sync-git-verify-commit-signature', 'false') != 'true'): return True @@ -180,7 +209,7 @@ class GitSync(NewBase): env = os.environ.copy() env['GNUPGHOME'] = openpgp_env.home - rev_cmd = [self.bin_command, "log", "--pretty=format:%G?", "-1"] +
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/sync/
commit: 82823b0c4de0a7cbb5654bb19d63aef874800afd Author: Zac Medico gentoo org> AuthorDate: Thu Jul 5 10:18:55 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Jul 5 10:19:23 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=82823b0c test_sync_local: fix GitSync coverage Fixes: 0655b4a26e37 ("test_sync_local: add test for auto-sync set to 'no'") pym/portage/tests/sync/test_sync_local.py | 15 +-- 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pym/portage/tests/sync/test_sync_local.py b/pym/portage/tests/sync/test_sync_local.py index 010c8f887..17ff6f200 100644 --- a/pym/portage/tests/sync/test_sync_local.py +++ b/pym/portage/tests/sync/test_sync_local.py @@ -102,17 +102,20 @@ class SyncLocalTestCase(TestCase): os.unlink(os.path.join(metadata_dir, 'timestamp.chk')) sync_cmds = ( + (homedir, cmds["emerge"] + ("--sync",)), + (homedir, lambda: self.assertTrue(os.path.exists( + os.path.join(repo.location, "dev-libs", "A") + ), "dev-libs/A expected, but missing")), + (homedir, cmds["emaint"] + ("sync", "-A")), + ) + + sync_cmds_auto_sync = ( (homedir, lambda: repos_set_conf("rsync", auto_sync="no")), (homedir, cmds["emerge"] + ("--sync",)), (homedir, lambda: self.assertFalse(os.path.exists( os.path.join(repo.location, "dev-libs", "A") ), "dev-libs/A found, expected missing")), (homedir, lambda: repos_set_conf("rsync", auto_sync="yes")), - (homedir, cmds["emerge"] + ("--sync",)), - (homedir, lambda: self.assertTrue(os.path.exists( - os.path.join(repo.location, "dev-libs", "A") - ), "dev-libs/A expected, but missing")), - (homedir, cmds["emaint"] + ("sync", "-A")), ) rename_repo = ( @@ -236,7 +239,7 @@ class SyncLocalTestCase(TestCase): # triggered by python -Wd will be visible. stdout = subprocess.PIPE - for cwd, cmd in rename_repo + sync_cmds + \ + for cwd, cmd in rename_repo + sync_cmds_auto_sync + sync_cmds + \ rsync_opts_repos + rsync_opts_repos_default + \ rsync_opts_repos_default_ovr + rsync_opts_repos_default_cancel + \ delete_sync_repo + git_repo_create + sync_type_git + \
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/, pym/portage/util/
commit: deb87a465306d05146d7eb55d27d7d89943725c0 Author: Zac Medico gentoo org> AuthorDate: Sun Jun 24 21:42:52 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed Jun 27 03:15:08 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=deb87a46 {,PKG_}INSTALL_MASK: support trailing slash (bug 658322) Fix the python INSTALL_MASK implementation so that a trailing slash matches a directory, for compatibility with the previous bash implementation. Fixes: 3416876c0ee7 ("{,PKG_}INSTALL_MASK: python implementation") Bug: https://bugs.gentoo.org/658322 pym/portage/tests/util/test_install_mask.py | 129 pym/portage/util/install_mask.py| 7 +- 2 files changed, 134 insertions(+), 2 deletions(-) diff --git a/pym/portage/tests/util/test_install_mask.py b/pym/portage/tests/util/test_install_mask.py new file mode 100644 index 0..f651eb4b7 --- /dev/null +++ b/pym/portage/tests/util/test_install_mask.py @@ -0,0 +1,129 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.util.install_mask import InstallMask + + +class InstallMaskTestCase(TestCase): + + def testTrailingSlash(self): + """ + Test that elements with a trailing slash match a directory + but not a regular file. + """ + cases = ( + ( + '/foo/bar/ -/foo/bar/*.foo -*.baz', + ( + ( + 'foo/bar/baz', + True, + ), + ( + 'foo/bar/', + True, + ), + # /foo/bar/ does not match + ( + 'foo/bar', + False, + ), + # this is excluded + ( + 'foo/bar/baz.foo', + False, + ), + # this is excluded + ( + 'foo/bar/baz.baz', + False, + ), + ( + 'foo/bar/baz.bar', + True, + ), + ) + ), + ( + '/foo/bar -/foo/bar/*.foo -*.baz', + ( + ( + 'foo/bar/baz', + True, + ), + # /foo/bar matches both foo/bar/ and foo/bar + ( + 'foo/bar/', + True, + ), + ( + 'foo/bar', + True, + ), + # this is excluded + ( + 'foo/bar/baz.foo', + False, + ), + # this is excluded + ( + 'foo/bar/baz.baz', + False, + ), + ( + 'foo/bar/baz.bar', + True, + ), + ) + ), + ( + '/foo*', + ( + ( + 'foo', + True, + ), +
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 6b50ba69f5a8e311fcddfb2e5c203631bd292c71 Author: Zac Medico gentoo org> AuthorDate: Thu Jun 21 21:03:38 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed Jun 27 03:04:38 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6b50ba69 AsyncioEventLoop: exit after unhandled exception (bug 658684) Fix portage commands to exit immediately for any unhandled exceptions that are raised while the asyncio event loop is running without a tty. If we have a tty then start the debugger, since in might aid in diagnosis of the problem. In order to avoid potential interference with API consumers, do not call set_exception_handler unless portage._internal_caller is True. Bug: https://bugs.gentoo.org/658684 pym/portage/util/_eventloop/asyncio_event_loop.py | 31 +++ 1 file changed, 31 insertions(+) diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py index c07b71103..ea0e03b23 100644 --- a/pym/portage/util/_eventloop/asyncio_event_loop.py +++ b/pym/portage/util/_eventloop/asyncio_event_loop.py @@ -2,7 +2,9 @@ # Distributed under the terms of the GNU General Public License v2 import os +import pdb import signal +import sys try: import asyncio as _real_asyncio @@ -53,6 +55,35 @@ class AsyncioEventLoop(_AbstractEventLoop): self.get_debug = loop.get_debug self._wakeup_fd = -1 + if portage._internal_caller: + loop.set_exception_handler(self._internal_caller_exception_handler) + + @staticmethod + def _internal_caller_exception_handler(loop, context): + """ + An exception handler which drops to a pdb shell if std* streams + refer to a tty, and otherwise kills the process with SIGTERM. + + In order to avoid potential interference with API consumers, this + implementation is only used when portage._internal_caller is True. + """ + loop.default_exception_handler(context) + if 'exception' in context: + # If we have a tty then start the debugger, since in might + # aid in diagnosis of the problem. If there's no tty, then + # exit immediately. + if all(s.isatty() for s in (sys.stdout, sys.stderr, sys.stdin)): + pdb.set_trace() + else: + # Normally emerge will wait for all coroutines to complete + # after SIGTERM has been received. However, an unhandled + # exception will prevent the interrupted coroutine from + # completing, therefore use the default SIGTERM handler + # in order to ensure that emerge exits immediately (though + # uncleanly). + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGTERM) + def _create_future(self): """ Provide AbstractEventLoop.create_future() for python3.4.
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
commit: 2710f9ab525e7c726c2ffb027e242dbdf31cfe76 Author: Zac Medico gentoo org> AuthorDate: Tue Jun 26 20:14:09 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue Jun 26 20:16:25 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2710f9ab binarytree._merge_pkgindex_header: deduplicate values (bug 657422) Deduplicate values of implicit IUSE variables, in order to prevent them from growing without bound, triggering "[Errno 7] Argument list too long" errors as reported in bug 657422. Fixes: cab78dba98c4 ("emerge --usepkgonly: propagate implicit IUSE and USE_EXPAND (bug 640318)") Bug: https://bugs.gentoo.org/657422 pym/portage/dbapi/bintree.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 269a7b226..9c2d877e7 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -1320,7 +1320,9 @@ class binarytree(object): for k, v in iter_iuse_vars(src): v_before = dest.get(k) if v_before is not None: - v = v_before + ' ' + v + merged_values = set(v_before.split()) + merged_values.update(v.split()) + v = ' '.join(sorted(merged_values)) dest[k] = v if 'ARCH' not in dest and 'ARCH' in src:
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/
commit: 294f1a18f3271f7d506d346c2a514a2afa6ce8ec Author: Ulrich Müller gentoo org> AuthorDate: Sat Jun 23 09:22:18 2018 + Commit: Ulrich Müller gentoo org> CommitDate: Sat Jun 23 09:22:18 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=294f1a18 Revert "_post_src_install_uid_fix: allow files with portage group permissions (bug 600804)" This reverts commit f479a4cdcac5db92231f489f232f10eb934c6f12. Acked by zmedico in #gentoo-portage. Bug: https://bugs.gentoo.org/600804 pym/portage/package/ebuild/doebuild.py | 9 +++-- 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index 0e94de805..97a9199a3 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -2045,7 +2045,7 @@ def _postinst_bsdflags(mysettings): def _post_src_install_uid_fix(mysettings, out): """ Files in $D with user and group bits that match the "portage" - user and group are automatically mapped to PORTAGE_INST_UID and + user or group are automatically mapped to PORTAGE_INST_UID and PORTAGE_INST_GID if necessary. The chown system call may clear S_ISUID and S_ISGID bits, so those bits are restored if necessary. @@ -2191,11 +2191,8 @@ def _post_src_install_uid_fix(mysettings, out): mystat.st_ino not in counted_inodes: counted_inodes.add(mystat.st_ino) size += mystat.st_size - - # Only remap the UID/GID if both match the portage user, - # in order to avoid interference with ebuilds that install - # files with portage group permissions (see bug 600804). - if (mystat.st_uid, mystat.st_gid) != (portage_uid, portage_gid): + if mystat.st_uid != portage_uid and \ + mystat.st_gid != portage_gid: continue myuid = -1 mygid = -1
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/, pym/_emerge/
commit: 345256c2d439c5ab580e4226f227db2819883d40 Author: Zac Medico gentoo org> AuthorDate: Mon Jun 4 23:32:46 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Jun 4 23:44:15 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=345256c2 emerge -pf: spawn pkg_nofetch asynchronously (bug 657360) For pretend mode, fix doebuild to skip the spawn_nofetch call that would trigger event loop recursion, and spawn pkg_nofetch asynchronously. Bug: https://bugs.gentoo.org/657360 pym/_emerge/EbuildBuild.py | 16 ++-- pym/_emerge/EbuildFetchonly.py | 5 - pym/portage/package/ebuild/doebuild.py | 15 +++ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index 00d4680f5..d9f7f6da7 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -142,8 +142,20 @@ class EbuildBuild(CompositeTask): pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() - self.returncode = retval - self.wait() + if retval == os.EX_OK: + self._current_task = None + self.returncode = os.EX_OK + self._async_wait() + else: + # For pretend mode, the convention it to execute + # pkg_nofetch and return a successful exitcode. + self._start_task(SpawnNofetchWithoutBuilddir( + background=self.background, + portdb=self.pkg.root_config.trees[self._tree].dbapi, + ebuild_path=self._ebuild_path, + scheduler=self.scheduler, + settings=self.settings), + self._default_final_exit) return else: fetcher = EbuildFetcher( diff --git a/pym/_emerge/EbuildFetchonly.py b/pym/_emerge/EbuildFetchonly.py index f88ea96ef..eec2ad208 100644 --- a/pym/_emerge/EbuildFetchonly.py +++ b/pym/_emerge/EbuildFetchonly.py @@ -25,7 +25,10 @@ class EbuildFetchonly(SlotObject): listonly=self.pretend, fetchonly=1, fetchall=self.fetch_all, mydbapi=portdb, tree="porttree") - if rval != os.EX_OK: + # For pretend mode, this error message is suppressed, + # and the unsuccessful return value is used to trigger + # a call to the pkg_nofetch phase. + if rval != os.EX_OK and not self.pretend: msg = "Fetch failed for '%s'" % (pkg.cpv,) eerror(msg, phase="unpack", key=pkg.cpv) diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index dc443df00..0e94de805 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -1079,14 +1079,13 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, if not fetch(fetchme, mysettings, listonly=listonly, fetchonly=fetchonly, allow_missing_digests=False, digests=dist_digests): - spawn_nofetch(mydbapi, myebuild, settings=mysettings, - fd_pipes=fd_pipes) - if listonly: - # The convention for listonly mode is to report - # success in any case, even though fetch() may - # return unsuccessfully in order to trigger the - # nofetch phase. - return 0 + # Since listonly mode is called by emerge --pretend in an + # asynchronous context, spawn_nofetch would trigger event loop + # recursion here, therefore delegate execution of pkg_nofetch + # to the caller (bug 657360). + if not listonly: + spawn_nofetch(mydbapi, myebuild, settings=mysettings, + fd_pipes=fd_pipes) return 1 if need_distfiles:
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/process/
commit: ac5b48b253add3007034f9fdad02779cd3972281 Author: Zac Medico gentoo org> AuthorDate: Sat May 26 22:38:58 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat May 26 22:38:58 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ac5b48b2 PipeReaderTestCase: wait for consumer pym/portage/tests/process/test_poll.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pym/portage/tests/process/test_poll.py b/pym/portage/tests/process/test_poll.py index d71c9b59c..f700a5585 100644 --- a/pym/portage/tests/process/test_poll.py +++ b/pym/portage/tests/process/test_poll.py @@ -78,6 +78,7 @@ class PipeReaderTestCase(TestCase): producer.start() os.close(slave_fd) producer.wait() + consumer.wait() self.assertEqual(producer.returncode, os.EX_OK) self.assertEqual(consumer.returncode, os.EX_OK)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 4fb5ef2ce2cb27ae155a25bfa5a4666597afb6ac Author: Zac Medico gentoo org> AuthorDate: Sat May 26 20:20:06 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat May 26 20:20:06 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4fb5ef2c EventLoop.close: fix 'AttributeError: close' for python2 For python2 without epoll, fix handling of missing 'close' attribute on self._poll_obj. Fixes: 4095be74985c ("Add ForkExecutor (bug 649588)") Reported-by: Brian Evans gentoo.org> pym/portage/util/_eventloop/EventLoop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index de0795224..df76374d9 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -968,7 +968,7 @@ class EventLoop(object): executor.shutdown(wait=True) if self._poll_obj is not None: - close = getattr(self._poll_obj, 'close') + close = getattr(self._poll_obj, 'close', None) if close is not None: close() self._poll_obj = None
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/process/
commit: 3b0c52b9ef364dc8e69208ec5341255ac94d41d8 Author: Zac Medico gentoo org> AuthorDate: Sat May 26 10:44:38 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat May 26 10:44:38 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3b0c52b9 PipeReaderTestCase: cover sockets and named pipes pym/portage/tests/process/test_poll.py | 74 ++ 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/pym/portage/tests/process/test_poll.py b/pym/portage/tests/process/test_poll.py index 596ea3088..d71c9b59c 100644 --- a/pym/portage/tests/process/test_poll.py +++ b/pym/portage/tests/process/test_poll.py @@ -1,11 +1,16 @@ -# Copyright 1998-2013 Gentoo Foundation +# Copyright 1998-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import functools +import pty +import shutil +import socket +import sys import subprocess +import tempfile from portage import os from portage.tests import TestCase -from portage.util._pty import _create_pty_or_pipe from portage.util._async.PopenProcess import PopenProcess from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.PipeReader import PipeReader @@ -13,28 +18,47 @@ from _emerge.PipeReader import PipeReader class PipeReaderTestCase(TestCase): _use_array = False - _use_pty = False _echo_cmd = "echo -n '%s'" - def _testPipeReader(self, test_string): + def test_pipe(self): + def make_pipes(): + return os.pipe(), None + self._do_test(make_pipes) + + def test_pty_device(self): + def make_pipes(): + try: + return pty.openpty(), None + except EnvironmentError: + self.skipTest('pty not available') + self._do_test(make_pipes) + + def test_domain_socket(self): + def make_pipes(): + if sys.version_info >= (3, 2): + read_end, write_end = socket.socketpair() + return (read_end.detach(), write_end.detach()), None + else: + self.skipTest('socket detach not supported') + self._do_test(make_pipes) + + def test_named_pipe(self): + def make_pipes(): + tempdir = tempfile.mkdtemp() + fifo_path = os.path.join(tempdir, 'fifo') + os.mkfifo(fifo_path) + return ((os.open(fifo_path, os.O_NONBLOCK|os.O_RDONLY), + os.open(fifo_path, os.O_NONBLOCK|os.O_WRONLY)), + functools.partial(shutil.rmtree, tempdir)) + self._do_test(make_pipes) + + def _testPipeReader(self, master_fd, slave_fd, test_string): """ Use a poll loop to read data from a pipe and assert that the data written to the pipe is identical to the data read from the pipe. """ - if self._use_pty: - got_pty, master_fd, slave_fd = _create_pty_or_pipe() - if not got_pty: - os.close(slave_fd) - os.close(master_fd) - skip_reason = "pty not acquired" - self.portage_skip = skip_reason - self.fail(skip_reason) - return - else: - master_fd, slave_fd = os.pipe() - # WARNING: It is very important to use unbuffered mode here, # in order to avoid issue 5380 with python3. master_file = os.fdopen(master_fd, 'rb', 0) @@ -60,15 +84,18 @@ class PipeReaderTestCase(TestCase): return consumer.getvalue().decode('ascii', 'replace') - def testPipeReader(self): + def _do_test(self, make_pipes): for x in (1, 2, 5, 6, 7, 8, 2**5, 2**10, 2**12, 2**13, 2**14): test_string = x * "a" - output = self._testPipeReader(test_string) - self.assertEqual(test_string, output, - "x = %s, len(output) = %s" % (x, len(output))) + (read_end, write_end), cleanup = make_pipes() + try: + output = self._testPipeReader(read_end, write_end, test_string) + self.assertEqual(test_string, output, + "x = %s, len(output) = %s" % (x, len(output))) + finally: + if cleanup is not None: + cleanup() -class
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/
commit: cd44fea4c8c8154fee74bcc37529c0d5c2e9d789 Author: Zac Medico gentoo org> AuthorDate: Sat May 26 05:46:36 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat May 26 06:07:55 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cd44fea4 prepare_build_dirs: convert PermissionError to PermissionDenied (bug 656542) This supresses display of a big PermissionError traceback for bin/ebuild, like in https://bugs.gentoo.org/656542#c0, and also suppresses additional messages that may occur when the unhandled PermissionError prevents bin/ebuild from cleanly closing the asyncio event loop. Bug: https://bugs.gentoo.org/656542 pym/portage/package/ebuild/prepare_build_dirs.py | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pym/portage/package/ebuild/prepare_build_dirs.py b/pym/portage/package/ebuild/prepare_build_dirs.py index 21e2aa87c..e53ccd0fb 100644 --- a/pym/portage/package/ebuild/prepare_build_dirs.py +++ b/pym/portage/package/ebuild/prepare_build_dirs.py @@ -19,6 +19,7 @@ from portage.localization import _ from portage.output import colorize from portage.util import apply_recursive_permissions, \ apply_secpass_permissions, ensure_dirs, normalize_path, writemsg +from portage.util.install_mask import _raise_exc from portage.const import EPREFIX def prepare_build_dirs(myroot=None, settings=None, cleanup=False): @@ -50,7 +51,9 @@ def prepare_build_dirs(myroot=None, settings=None, cleanup=False): clean_dir, noiselevel=-1) return 1 else: - raise + # Wrap with PermissionDenied if appropriate, so that callers + # display a short error message without a traceback. + _raise_exc(oe) def makedirs(dir_path): try:
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
commit: 466cc6c60fcf1d56049af7a1d37fc959f6808351 Author: Zac Medico gentoo org> AuthorDate: Fri May 25 16:07:33 2018 + Commit: Zac Medico gentoo org> CommitDate: Fri May 25 16:10:34 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=466cc6c6 SonameDepsProcessor: fix 'set' object has no attribute 'items' (bug 656492) Use collections.defaultdict to ensure that requires_map contains correct defaults. Fixes: 1364cd44e7a6 ("SonameDepsProcessor: handle internal libs without DT_SONAME (bug 646190)") Bug: https://bugs.gentoo.org/656492 pym/portage/util/_dyn_libs/soname_deps.py | 9 + 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pym/portage/util/_dyn_libs/soname_deps.py b/pym/portage/util/_dyn_libs/soname_deps.py index c6302afc2..544cbc8f1 100644 --- a/pym/portage/util/_dyn_libs/soname_deps.py +++ b/pym/portage/util/_dyn_libs/soname_deps.py @@ -3,7 +3,9 @@ from __future__ import unicode_literals +import collections import fnmatch +import functools from itertools import chain import os import re @@ -33,7 +35,8 @@ class SonameDepsProcessor(object): """ self._provides_exclude = self._exclude_pattern(provides_exclude) self._requires_exclude = self._exclude_pattern(requires_exclude) - self._requires_map = {} + self._requires_map = collections.defaultdict( + functools.partial(collections.defaultdict, set)) self._provides_map = {} self._provides_unfiltered = {} self._basename_map = {} @@ -84,8 +87,7 @@ class SonameDepsProcessor(object): for x in entry.needed: if (self._requires_exclude is None or self._requires_exclude.match(x) is None): - self._requires_map.setdefault( - multilib_cat, {}).setdefault(x, set()).add(runpaths) + self._requires_map[multilib_cat][x].add(runpaths) if entry.soname: self._provides_unfiltered.setdefault( @@ -105,7 +107,6 @@ class SonameDepsProcessor(object): provides_unfiltered = self._provides_unfiltered for multilib_cat in set(chain(requires_map, provides_map)): - requires_map.setdefault(multilib_cat, set()) provides_map.setdefault(multilib_cat, set()) provides_unfiltered.setdefault(multilib_cat, set()) for soname, consumers in list(requires_map[multilib_cat].items()):
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 8a8f527c7587bf388645ad703e0305797a26c3b4 Author: Zac Medico gentoo org> AuthorDate: Fri May 25 03:06:08 2018 + Commit: Zac Medico gentoo org> CommitDate: Fri May 25 03:06:32 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8a8f527c AsyncioEventLoop: remove redundant set_wakeup_fd call pym/portage/util/_eventloop/asyncio_event_loop.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py index 65b354544..c07b71103 100644 --- a/pym/portage/util/_eventloop/asyncio_event_loop.py +++ b/pym/portage/util/_eventloop/asyncio_event_loop.py @@ -104,5 +104,3 @@ class AsyncioEventLoop(_AbstractEventLoop): return self._loop.run_until_complete(future) finally: self._wakeup_fd = signal.set_wakeup_fd(-1) - if self._wakeup_fd != -1: - signal.set_wakeup_fd(-1)
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/, pym/portage/util/_eventloop/
commit: adee194534f0b3d9762efd1e8e8713c316b93f5a Author: Zac Medico gentoo org> AuthorDate: Thu May 24 22:36:29 2018 + Commit: Zac Medico gentoo org> CommitDate: Fri May 25 02:01:27 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=adee1945 AsyncioEventLoop: suppress BlockingIOError warning (bug 655656) Override AbstractEventLoop.run_until_complete() to prevent BlockingIOError from occurring when the event loop is not running, by using signal.set_wakeup_fd(-1) to temporarily disable the wakeup fd. In order to avoid potential interference with API consumers, only modify wakeup fd when portage._interal_caller is True. Bug: https://bugs.gentoo.org/655656 .../util/futures/asyncio/test_wakeup_fd_sigchld.py | 76 ++ pym/portage/util/_eventloop/asyncio_event_loop.py | 37 +-- 2 files changed, 106 insertions(+), 7 deletions(-) diff --git a/pym/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py b/pym/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py new file mode 100644 index 0..abc67c241 --- /dev/null +++ b/pym/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py @@ -0,0 +1,76 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import os +import subprocess + +import portage +from portage.const import PORTAGE_PYM_PATH +from portage.tests import TestCase +from portage.util._eventloop.global_event_loop import _asyncio_enabled + + +class WakeupFdSigchldTestCase(TestCase): + def testWakeupFdSigchld(self): + """ + This is expected to trigger a bunch of messages like the following + unless the fix for bug 655656 works as intended: + + Exception ignored when trying to write to the signal wakeup fd: + BlockingIOError: [Errno 11] Resource temporarily unavailable + """ + if not _asyncio_enabled: + self.skipTest('asyncio not enabled') + + script = """ +import asyncio as _real_asyncio +import os +import signal +import sys + +import portage + +# In order to avoid potential interference with API consumers, wakeup +# fd handling is enabled only when portage._interal_caller is True. +portage._internal_caller = True + +from portage.util.futures import asyncio + +loop = asyncio._wrap_loop() + +# Cause the loop to register a child watcher. +proc = loop.run_until_complete(_real_asyncio.create_subprocess_exec('sleep', '0')) +loop.run_until_complete(proc.wait()) + +for i in range(8192): + os.kill(os.getpid(), signal.SIGCHLD) + +# Verify that the child watcher still works correctly +# (this will hang if it doesn't). +proc = loop.run_until_complete(_real_asyncio.create_subprocess_exec('sleep', '0')) +loop.run_until_complete(proc.wait()) +loop.close() +sys.stdout.write('success') +sys.exit(os.EX_OK) +""" + + pythonpath = os.environ.get('PYTHONPATH', '').strip().split(':') + if not pythonpath or pythonpath[0] != PORTAGE_PYM_PATH: + pythonpath = [PORTAGE_PYM_PATH] + pythonpath + pythonpath = ':'.join(filter(None, pythonpath)) + + proc = subprocess.Popen( + [portage._python_interpreter, '-c', script], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=dict(os.environ, PYTHONPATH=pythonpath)) + + out, err = proc.communicate() + try: + self.assertEqual(out[:100], b'success') + except Exception: + portage.writemsg(''.join('{}\n'.format(line) + for line in out.decode(errors='replace').splitlines()[:50]), + noiselevel=-1) + raise + + self.assertEqual(proc.wait(), os.EX_OK) diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py index bf5937de8..65b354544 100644 --- a/pym/portage/util/_eventloop/asyncio_event_loop.py +++ b/pym/portage/util/_eventloop/asyncio_event_loop.py @@ -1,6 +1,7 @@ # Copyright 2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import os import signal try: @@ -11,6 +12,8 @@ except ImportError: _real_asyncio = None _AbstractEventLoop = object +import portage + class AsyncioEventLoop(_AbstractEventLoop): """ @@ -26,13 +29,15 @@ class AsyncioEventLoop(_AbstractEventLoop): def __init__(self, loop=None): loop = loop or _real_asyncio.get_event_loop() self._loop = loop - self.run_until_complete = loop.run_until_complete + self.run_until_complete = (self._run_until_complete + if portage._internal_caller else loop.run_until_complete) self.call_soo
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/, pym/portage/tests/util/dyn_libs/
commit: 1364cd44e7a6232bf425c4573b5bd3d6738d49a4 Author: Zac Medico gentoo org> AuthorDate: Sun May 20 06:40:19 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu May 24 20:31:02 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1364cd44 SonameDepsProcessor: handle internal libs without DT_SONAME (bug 646190) Packages like ebtables have internal libraries that lack a DT_SONAME field in their ELF header. Consumers of these internal libraries have DT_RUNPATH entries that refer to the directory containing the internal libraries. For library dependencies that are satisfied by internal libraries like this, it is inappropriate for SonameDepsProcessor to include these depenedencies in the REQUIRES metadata, therefore fix SonameDepsProcessor to automatically detect this case and exclude these dependencies from the REQUIRES metadata. This solves incorrect reporting of broken soname dependencies like the following: $ emerge -p --depclean --ignore-soname-deps=n Calculating dependencies... done! * Broken soname dependencies found: * * x86_64: libebt_redirect.so required by: * net-firewall/ebtables-2.0.10.4 * * x86_64: libebt_log.so required by: * net-firewall/ebtables-2.0.10.4 Bug: https://bugs.gentoo.org/646190 pym/portage/tests/util/dyn_libs/__init__.py| 0 pym/portage/tests/util/dyn_libs/__test__.py| 0 .../tests/util/dyn_libs/test_soname_deps.py| 34 + pym/portage/util/_dyn_libs/soname_deps.py | 35 -- 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/pym/portage/tests/util/dyn_libs/__init__.py b/pym/portage/tests/util/dyn_libs/__init__.py new file mode 100644 index 0..e69de29bb diff --git a/pym/portage/tests/util/dyn_libs/__test__.py b/pym/portage/tests/util/dyn_libs/__test__.py new file mode 100644 index 0..e69de29bb diff --git a/pym/portage/tests/util/dyn_libs/test_soname_deps.py b/pym/portage/tests/util/dyn_libs/test_soname_deps.py new file mode 100644 index 0..823890c91 --- /dev/null +++ b/pym/portage/tests/util/dyn_libs/test_soname_deps.py @@ -0,0 +1,34 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.util._dyn_libs.NeededEntry import NeededEntry +from portage.util._dyn_libs.soname_deps import SonameDepsProcessor + + +class SonameDepsProcessorTestCase(TestCase): + + def testInternalLibsWithoutSoname(self): + """ + Test handling of internal libraries that lack an soname, which are + resolved via DT_RUNPATH, see ebtables for example (bug 646190). + """ + needed_elf_2 = """ +X86_64;/sbin/ebtables;;/lib64/ebtables;libebt_802_3.so,libebtable_broute.so,libc.so.6;x86_64 +X86_64;/lib64/ebtables/libebtable_broute.so;;;libc.so.6;x86_64 +X86_64;/lib64/ebtables/libebt_802_3.so;;;libc.so.6;x86_64 +""" + soname_deps = SonameDepsProcessor('', '') + + for line in needed_elf_2.splitlines(): + if not line: + continue + entry = NeededEntry.parse(None, line) + soname_deps.add(entry) + + self.assertEqual(soname_deps.provides, None) + # Prior to the fix for bug 646190, REQUIRES contained references to + # the internal libebt* libraries which are resolved via a DT_RUNPATH + # entry referring to the /lib64/ebtables directory that contains the + # internal libraries. + self.assertEqual(soname_deps.requires, 'x86_64: libc.so.6\n') diff --git a/pym/portage/util/_dyn_libs/soname_deps.py b/pym/portage/util/_dyn_libs/soname_deps.py index a7d595429..c6302afc2 100644 --- a/pym/portage/util/_dyn_libs/soname_deps.py +++ b/pym/portage/util/_dyn_libs/soname_deps.py @@ -9,6 +9,11 @@ import os import re from portage.util import shlex_split +from portage.util import ( + normalize_path, + varexpand, +) + class SonameDepsProcessor(object): """ @@ -31,6 +36,7 @@ class SonameDepsProcessor(object): self._requires_map = {} self._provides_map = {} self._provides_unfiltered = {} + self._basename_map = {} self._provides = None self._requires = None self._intersected = False @@ -62,15 +68,24 @@ class SonameDepsProcessor(object): raise AssertionError( "Missing multilib category data: %s" % entry.filename) + self._basename_map.setdefault( + os.path.basename(entry.filename), []).append(entry) + if entry.needed and ( self._requires_exclude is None or self._requires_exclude.match(
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/
commit: 28d3c9ed46b16757626b292dd6d7a6cae054b669 Author: Zac Medico gentoo org> AuthorDate: Thu May 24 19:40:10 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu May 24 19:42:13 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=28d3c9ed doebuild: eliminate redundant aux_get calls (bug 656394) These redundant aux_get calls triggered event loop recursion when called via EbuildFetchonly as reported in bug 656394. Bug: https://bugs.gentoo.org/656394 pym/portage/package/ebuild/doebuild.py | 19 +-- 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index c3b89ade2..dc443df00 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -1048,9 +1048,11 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, mydo not in ("digest", "manifest") and "noauto" not in features) if need_distfiles: - src_uri, = mydbapi.aux_get(mysettings.mycpv, - ["SRC_URI"], mytree=os.path.dirname(os.path.dirname( - os.path.dirname(myebuild + src_uri = mysettings.configdict["pkg"].get("SRC_URI") + if src_uri is None: + src_uri, = mydbapi.aux_get(mysettings.mycpv, + ["SRC_URI"], mytree=os.path.dirname(os.path.dirname( + os.path.dirname(myebuild metadata = { "EAPI": mysettings["EAPI"], "SRC_URI" : src_uri, @@ -1410,9 +1412,14 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): all_keys = set(Package.metadata_keys) all_keys.add("SRC_URI") all_keys = tuple(all_keys) - metadata = dict(zip(all_keys, - mydbapi.aux_get(mysettings.mycpv, all_keys, - myrepo=mysettings.get("PORTAGE_REPO_NAME" + metadata = mysettings.configdict['pkg'] + if all(k in metadata for k in ("PORTAGE_REPO_NAME", "SRC_URI")): + metadata = dict(((k, metadata[k]) for k in all_keys if k in metadata), + repository=metadata["PORTAGE_REPO_NAME"]) + else: + metadata = dict(zip(all_keys, + mydbapi.aux_get(mysettings.mycpv, all_keys, + myrepo=mysettings.get("PORTAGE_REPO_NAME" class FakeTree(object): def __init__(self, mydb):
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/portage/
commit: b2a5f03abc0c172b6189226e6f9a7491a002cf51 Author: Zac Medico gentoo org> AuthorDate: Thu May 17 18:38:27 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu May 17 18:38:27 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2a5f03a MergeProcess,spawn: unregister SIGCHLD and wakeup_fd (bug 655656) In order to prevent forked processes from invoking the parent process's SIGCHLD handler and writing to wakeup_fd (triggering BlockingIOError), unregister the SIGCHLD and wakeup_fd. Bug: https://bugs.gentoo.org/655656 Reported-by: Helmut Jarausch igpm.rwth-aachen.de> pym/portage/dbapi/_MergeProcess.py | 10 ++ pym/portage/process.py | 10 ++ 2 files changed, 20 insertions(+) diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py index fefdf8635..371550079 100644 --- a/pym/portage/dbapi/_MergeProcess.py +++ b/pym/portage/dbapi/_MergeProcess.py @@ -178,6 +178,16 @@ class MergeProcess(ForkProcess): signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) + # Unregister SIGCHLD handler and wakeup_fd for the parent + # process's event loop (bug 655656). + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + try: + wakeup_fd = signal.set_wakeup_fd(-1) + if wakeup_fd > 0: + os.close(wakeup_fd) + except (ValueError, OSError): + pass + portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). diff --git a/pym/portage/process.py b/pym/portage/process.py index 2af783e22..fd326731a 100644 --- a/pym/portage/process.py +++ b/pym/portage/process.py @@ -472,6 +472,16 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) + # Unregister SIGCHLD handler and wakeup_fd for the parent + # process's event loop (bug 655656). + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + try: + wakeup_fd = signal.set_wakeup_fd(-1) + if wakeup_fd > 0: + os.close(wakeup_fd) + except (ValueError, OSError): + pass + # Quiet killing of subprocesses by SIGPIPE (see bug #309001). signal.signal(signal.SIGPIPE, signal.SIG_DFL)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: b28b01a78e4cc0199111e804af5b56a33319356d Author: Zac Medico gentoo org> AuthorDate: Mon May 14 15:33:15 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon May 14 15:35:53 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b28b01a7 ForkProcess: unregister SIGCHLD and wakeup_fd (bug 655656) In order to prevent forked processes from invoking the parent process's SIGCHLD handler and writing to wakeup_fd (triggering BlockingIOError), unregister the SIGCHLD and wakeup_fd. Bug: https://bugs.gentoo.org/655656 Reported-by: Helmut Jarausch igpm.rwth-aachen.de> pym/portage/util/_async/ForkProcess.py | 10 ++ 1 file changed, 10 insertions(+) diff --git a/pym/portage/util/_async/ForkProcess.py b/pym/portage/util/_async/ForkProcess.py index 25f72d308..d84e93833 100644 --- a/pym/portage/util/_async/ForkProcess.py +++ b/pym/portage/util/_async/ForkProcess.py @@ -37,6 +37,16 @@ class ForkProcess(SpawnProcess): signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) + # Unregister SIGCHLD handler and wakeup_fd for the parent + # process's event loop (bug 655656). + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + try: + wakeup_fd = signal.set_wakeup_fd(-1) + if wakeup_fd > 0: + os.close(wakeup_fd) + except (ValueError, OSError): + pass + portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring).
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 65379f436759dfbc4d56e52f1a145950779ebb60 Author: Zac Medico gentoo org> AuthorDate: Sun May 13 16:45:27 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun May 13 16:57:38 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=65379f43 AsyncioEventLoop: suppress BlockingIOError warning during loop close (bug 655656) Disable the asyncio event loop's SIGCHLD handler before attempting to close it, in order to suppress a harmless BlockingIOError warning during loop close. Closes: https://bugs.gentoo.org/655656 Reported-by: Helmut Jarausch igpm.rwth-aachen.de> pym/portage/util/_eventloop/asyncio_event_loop.py | 10 +- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py index b365939b0..bf5937de8 100644 --- a/pym/portage/util/_eventloop/asyncio_event_loop.py +++ b/pym/portage/util/_eventloop/asyncio_event_loop.py @@ -1,6 +1,8 @@ # Copyright 2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import signal + try: import asyncio as _real_asyncio from asyncio.events import AbstractEventLoop as _AbstractEventLoop @@ -31,7 +33,6 @@ class AsyncioEventLoop(_AbstractEventLoop): self.call_at = loop.call_at self.is_running = loop.is_running self.is_closed = loop.is_closed - self.close = loop.close self.create_future = (loop.create_future if hasattr(loop, 'create_future') else self._create_future) self.create_task = loop.create_task @@ -75,3 +76,10 @@ class AsyncioEventLoop(_AbstractEventLoop): @return: the internal event loop's AbstractEventLoop interface """ return self + + def close(self): + # Suppress spurious error messages like the following for bug 655656: + # Exception ignored when trying to write to the signal wakeup fd: + # BlockingIOError: [Errno 11] Resource temporarily unavailable + self._loop.remove_signal_handler(signal.SIGCHLD) + self._loop.close()
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/
commit: 3a55ecd1f79c31f477d7bdd0b9f0e97d8a15eb9e Author: Zac Medico gentoo org> AuthorDate: Wed May 9 14:19:11 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed May 9 14:19:11 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3a55ecd1 DefaultEventLoopPolicy: test NotImplementedError due to recursion .../asyncio/test_policy_wrapper_recursion.py | 29 ++ 1 file changed, 29 insertions(+) diff --git a/pym/portage/tests/util/futures/asyncio/test_policy_wrapper_recursion.py b/pym/portage/tests/util/futures/asyncio/test_policy_wrapper_recursion.py new file mode 100644 index 0..d3cd94b35 --- /dev/null +++ b/pym/portage/tests/util/futures/asyncio/test_policy_wrapper_recursion.py @@ -0,0 +1,29 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +try: + import asyncio +except ImportError: + asyncio = None + +from portage.tests import TestCase +from portage.util.futures.unix_events import DefaultEventLoopPolicy + + +class PolicyWrapperRecursionTestCase(TestCase): + def testPolicyWrapperRecursion(self): + if asyncio is None: + self.skipTest('asyncio is not available') + + initial_policy = asyncio.get_event_loop_policy() + if not isinstance(initial_policy, DefaultEventLoopPolicy): + asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) + + try: + with self.assertRaises(NotImplementedError): + asyncio.get_event_loop() + + with self.assertRaises(NotImplementedError): + asyncio.get_child_watcher() + finally: + asyncio.set_event_loop_policy(initial_policy)
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/
commit: 0c96d2d0b18036cb94d68c42783441d32af2d9d3 Author: Zac Medico gentoo org> AuthorDate: Wed May 9 07:54:26 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed May 9 07:59:56 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0c96d2d0 SubprocessExecTestCase: fix unintended skipTest The internal asyncio shim does not provide a create_subprocess_exec attribute, so access it directly from the real asyncio module. Fixes 82a3cda6f1ff ("Minimize _asyncio_wrapper usage (bug 654390)") .../util/futures/asyncio/test_subprocess_exec.py| 21 + 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py index 534d79c53..5a812ba6a 100644 --- a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py +++ b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py @@ -4,6 +4,11 @@ import os import subprocess +try: + from asyncio import create_subprocess_exec +except ImportError: + create_subprocess_exec = None + from portage.process import find_binary from portage.tests import TestCase from portage.util._eventloop.global_event_loop import global_event_loop @@ -71,7 +76,7 @@ class SubprocessExecTestCase(TestCase): self.assertFalse(global_event_loop().is_closed()) def testEcho(self): - if not hasattr(asyncio, 'create_subprocess_exec'): + if create_subprocess_exec is None: self.skipTest('create_subprocess_exec not implemented for python2') args_tuple = (b'hello', b'world') @@ -91,7 +96,7 @@ class SubprocessExecTestCase(TestCase): try: with open(os.devnull, 'rb', 0) as devnull: proc = loop.run_until_complete( - asyncio.create_subprocess_exec( + create_subprocess_exec( echo_binary, *args_tuple, stdin=devnull, stdout=stdout_pw, stderr=stdout_pw)) @@ -114,7 +119,7 @@ class SubprocessExecTestCase(TestCase): self._run_test(test) def testCat(self): - if not hasattr(asyncio, 'create_subprocess_exec'): + if create_subprocess_exec is None: self.skipTest('create_subprocess_exec not implemented for python2') stdin_data = b'hello world' @@ -138,7 +143,7 @@ class SubprocessExecTestCase(TestCase): output = None try: proc = loop.run_until_complete( - asyncio.create_subprocess_exec( + create_subprocess_exec( cat_binary, stdin=stdin_pr, stdout=stdout_pw, stderr=stdout_pw)) @@ -173,7 +178,7 @@ class SubprocessExecTestCase(TestCase): requires an AbstractEventLoop.connect_read_pipe implementation (and a ReadTransport implementation for it to return). """ - if not hasattr(asyncio, 'create_subprocess_exec'): + if create_subprocess_exec is None: self.skipTest('create_subprocess_exec not implemented for python2') args_tuple = (b'hello', b'world') @@ -184,7 +189,7 @@ class SubprocessExecTestCase(TestCase): def test(loop): with open(os.devnull, 'rb', 0) as devnull: proc = loop.run_until_complete( - asyncio.create_subprocess_exec( + create_subprocess_exec( echo_binary, *args_tuple, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)) @@ -202,7 +207,7 @@ class SubprocessExecTestCase(TestCase): requires an AbstractEventLoop.connect_write_pipe implementation (and a WriteTransport implementation for it to return). """ - if not hasattr(asyncio, 'create_subprocess_exec'): + if create_subprocess_exec is None: self.skipTest('create_subprocess_exec not implemented for python2') stdin_data = b'hello world' @@ -212,7 +217,7 @@ class SubprocessExecTestCase(TestCase): def test(loop): proc = loop.run_until_complete( - asyncio.create_subprocess_exec( + create_subprocess_exec(
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/_asyncio/, pym/portage/util/futures/
commit: 5e628787e6f4c720680aeeb8beeac88e37988a9e Author: Zac Medico gentoo org> AuthorDate: Wed May 9 07:33:37 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed May 9 07:40:34 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5e628787 DefaultEventLoopPolicy: raise NotImplementedError, not RecursionError Since the DefaultEventLoopPolicy wraps the underlying asyncio event loop policy, raise NotImplementedError if the current instance is set as the underlying event loop policy. This avoids a RecursionError that would flood the terminal with a large stack trace. pym/portage/util/futures/_asyncio/__init__.py | 6 +++--- pym/portage/util/futures/unix_events.py | 23 ++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/pym/portage/util/futures/_asyncio/__init__.py b/pym/portage/util/futures/_asyncio/__init__.py index 940da4762..acfd59396 100644 --- a/pym/portage/util/futures/_asyncio/__init__.py +++ b/pym/portage/util/futures/_asyncio/__init__.py @@ -32,7 +32,7 @@ except ImportError: import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.util.futures.unix_events:DefaultEventLoopPolicy', + 'portage.util.futures.unix_events:_PortageEventLoopPolicy', ) from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as _AsyncioEventLoop from portage.util._eventloop.global_event_loop import ( @@ -67,7 +67,7 @@ def get_event_loop_policy(): global _lock, _policy with _lock: if _policy is None: - _policy = DefaultEventLoopPolicy() + _policy = _PortageEventLoopPolicy() return _policy @@ -81,7 +81,7 @@ def set_event_loop_policy(policy): """ global _lock, _policy with _lock: - _policy = policy or DefaultEventLoopPolicy() + _policy = policy or _PortageEventLoopPolicy() def get_event_loop(): diff --git a/pym/portage/util/futures/unix_events.py b/pym/portage/util/futures/unix_events.py index 8eb369f8b..3381eaa7d 100644 --- a/pym/portage/util/futures/unix_events.py +++ b/pym/portage/util/futures/unix_events.py @@ -681,4 +681,25 @@ class _PortageEventLoopPolicy(events.AbstractEventLoopPolicy): return _global_event_loop()._asyncio_child_watcher -DefaultEventLoopPolicy = _PortageEventLoopPolicy +class _AsyncioEventLoopPolicy(_PortageEventLoopPolicy): + """ + A subclass of _PortageEventLoopPolicy which raises + NotImplementedError if it is set as the real asyncio event loop + policy, since this class is intended to *wrap* the real asyncio + event loop policy. + """ + def _check_recursion(self): + if _real_asyncio.get_event_loop_policy() is self: + raise NotImplementedError('this class is only a wrapper') + + def get_event_loop(self): + self._check_recursion() + return super(_AsyncioEventLoopPolicy, self).get_event_loop() + + def get_child_watcher(self): + self._check_recursion() + return super(_AsyncioEventLoopPolicy, self).get_child_watcher() + + +DefaultEventLoopPolicy = (_AsyncioEventLoopPolicy if _asyncio_enabled + else _PortageEventLoopPolicy)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/_asyncio/, pym/portage/util/futures/
commit: 920b90fd0883dbd36f0290d08c9af49a208c2950 Author: Zac Medico gentoo org> AuthorDate: Wed May 9 04:59:59 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed May 9 04:59:59 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=920b90fd _wrap_loop: default to global_event_loop behavior The default loop returned by _wrap_loop should be consistent with global_event_loop, in order to avoid accidental registration of callbacks with a loop that is not intended to run. Fixes 96cc07326391 ("global_event_loop: use asyncio event loop (bug 654390)") pym/portage/util/futures/_asyncio/__init__.py | 14 ++ pym/portage/util/futures/unix_events.py | 40 +-- 2 files changed, 10 insertions(+), 44 deletions(-) diff --git a/pym/portage/util/futures/_asyncio/__init__.py b/pym/portage/util/futures/_asyncio/__init__.py index 1273afa02..940da4762 100644 --- a/pym/portage/util/futures/_asyncio/__init__.py +++ b/pym/portage/util/futures/_asyncio/__init__.py @@ -35,7 +35,10 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.futures.unix_events:DefaultEventLoopPolicy', ) from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as _AsyncioEventLoop -from portage.util._eventloop.global_event_loop import _asyncio_enabled +from portage.util._eventloop.global_event_loop import ( + _asyncio_enabled, + global_event_loop as _global_event_loop, +) from portage.util.futures.futures import ( CancelledError, Future, @@ -168,14 +171,15 @@ def _wrap_loop(loop=None): @rtype: asyncio.AbstractEventLoop (or compatible) @return: event loop """ - return loop or get_event_loop() + return loop or _global_event_loop() if _asyncio_enabled: - get_event_loop_policy = _real_asyncio.get_event_loop_policy - set_event_loop_policy = _real_asyncio.set_event_loop_policy + # The default loop returned by _wrap_loop should be consistent + # with global_event_loop, in order to avoid accidental registration + # of callbacks with a loop that is not intended to run. def _wrap_loop(loop=None): - loop = loop or get_event_loop() + loop = loop or _global_event_loop() return (loop if hasattr(loop, '_asyncio_wrapper') else _AsyncioEventLoop(loop=loop)) diff --git a/pym/portage/util/futures/unix_events.py b/pym/portage/util/futures/unix_events.py index ce520db00..8eb369f8b 100644 --- a/pym/portage/util/futures/unix_events.py +++ b/pym/portage/util/futures/unix_events.py @@ -681,42 +681,4 @@ class _PortageEventLoopPolicy(events.AbstractEventLoopPolicy): return _global_event_loop()._asyncio_child_watcher -class _AsyncioEventLoopPolicy(_PortageEventLoopPolicy): - """ - Implementation of asyncio.AbstractEventLoopPolicy based on asyncio's - event loop. This supports running event loops in forks, - which is not supported by the default asyncio event loop policy, - see https://bugs.python.org/issue22087 and also - https://bugs.python.org/issue29703 which affects pypy3-5.10.1. - """ - _MAIN_PID = os.getpid() - - def __init__(self): - super(_AsyncioEventLoopPolicy, self).__init__() - self._default_policy = _real_asyncio.DefaultEventLoopPolicy() - - def get_event_loop(self): - """ - Get the event loop for the current context. - - Returns an event loop object implementing the AbstractEventLoop - interface. - - @rtype: asyncio.AbstractEventLoop (or compatible) - @return: the current event loop policy - """ - if os.getpid() == self._MAIN_PID: - return self._default_policy.get_event_loop() - else: - return super(_AsyncioEventLoopPolicy, self).get_event_loop() - - def get_child_watcher(self): - """Get the watcher for child processes.""" - if os.getpid() == self._MAIN_PID: - return self._default_policy.get_child_watcher() - else: - return super(_AsyncioEventLoopPolicy, self).get_child_watcher() - - -DefaultEventLoopPolicy = (_AsyncioEventLoopPolicy if _asyncio_enabled - else _PortageEventLoopPolicy) +DefaultEventLoopPolicy = _PortageEventLoopPolicy
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, pym/portage/util/_eventloop/, ...
commit: 96cc073263910e7c1b0f48cc08a4db4b56ffe408 Author: Zac Medico gentoo org> AuthorDate: Tue May 1 08:15:39 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue May 8 16:42:03 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=96cc0732 global_event_loop: use asyncio event loop (bug 654390) For python3.4 and later, in the main process, replace portage's internal event loop with the standard library's asyncio event loop. Continue to use portage's internal event loop in subprocesses, since asyncio's event loop is not guaranteed to work well in subprocesses (see upstream python issues 22087 and 29703). An _AsyncioEventLoopPolicy class, derived from _PortageEventLoopPolicy, is needed for some unit tests that modify asyncio's event loop policy. This policy is not needed for anything other than unit testing. Portage uses asyncio's default event loop policy, and API consumers are free to use any desired event loop policy. Portage's asynchronous functions that accept a 'loop' parameter will work with any compatible asyncio.AbstractEventLoop implementation, since an internal _wrap_loop function automatically adapts the loop for internal use. Bug: https://bugs.gentoo.org/654390 Reviewed-by: Brian Dolbec gentoo.org> Reviewed-by: Alec Warner gentoo.org> pym/portage/util/_async/SchedulerInterface.py | 3 + pym/portage/util/_eventloop/asyncio_event_loop.py | 77 +++ pym/portage/util/_eventloop/global_event_loop.py | 5 +- pym/portage/util/futures/_asyncio/__init__.py | 17 + pym/portage/util/futures/unix_events.py | 43 - 5 files changed, 143 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/_async/SchedulerInterface.py b/pym/portage/util/_async/SchedulerInterface.py index f1a3e9b0b..ec6417da1 100644 --- a/pym/portage/util/_async/SchedulerInterface.py +++ b/pym/portage/util/_async/SchedulerInterface.py @@ -33,6 +33,9 @@ class SchedulerInterface(SlotObject): "time", "_asyncio_child_watcher", + # This attribute it used by _wrap_loop to detect if the + # loop already has a suitable wrapper. + "_asyncio_wrapper", ) __slots__ = _event_loop_attrs + ("_event_loop", "_is_background") diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py new file mode 100644 index 0..b365939b0 --- /dev/null +++ b/pym/portage/util/_eventloop/asyncio_event_loop.py @@ -0,0 +1,77 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +try: + import asyncio as _real_asyncio + from asyncio.events import AbstractEventLoop as _AbstractEventLoop +except ImportError: + # Allow ImportModulesTestCase to succeed. + _real_asyncio = None + _AbstractEventLoop = object + + +class AsyncioEventLoop(_AbstractEventLoop): + """ + Implementation of asyncio.AbstractEventLoop which wraps asyncio's + event loop and is minimally compatible with _PortageEventLoop. + """ + + # Use portage's internal event loop in subprocesses, as a workaround + # for https://bugs.python.org/issue22087, and also + # https://bugs.python.org/issue29703 which affects pypy3-5.10.1. + supports_multiprocessing = False + + def __init__(self, loop=None): + loop = loop or _real_asyncio.get_event_loop() + self._loop = loop + self.run_until_complete = loop.run_until_complete + self.call_soon = loop.call_soon + self.call_soon_threadsafe = loop.call_soon_threadsafe + self.call_later = loop.call_later + self.call_at = loop.call_at + self.is_running = loop.is_running + self.is_closed = loop.is_closed + self.close = loop.close + self.create_future = (loop.create_future + if hasattr(loop, 'create_future') else self._create_future) + self.create_task = loop.create_task + self.add_reader = loop.add_reader + self.remove_reader = loop.remove_reader + self.add_writer = loop.add_writer + self.remove_writer = loop.remove_writer + self.run_in_executor = loop.run_in_executor + self.time = loop.time + self.default_exception_handler = loop.default_exception_handler + self.call_exception_handler = loop.call_exception_handler + self.set_debug = loop.set_debug + self.get_debug = loop.get_debug + + def _create_future(self): + """ + Provide AbstractEventLoop.create_future() for python3.4. + """ + return _real_asyncio.Future(loop=self._loop) + + @property + def _asyncio_child_watcher(self): +
[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/
commit: 505707a335683330ac69b1c7f428e8a1c66d0b2e Author: Zac Medico gentoo org> AuthorDate: Mon May 7 08:14:19 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon May 7 08:17:18 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=505707a3 rsync: explicitly use ForkExecutor for key refresh retry (bug 654390) The ThreadPoolExecutor that asyncio uses by default does not support cancellation of tasks, therefore use ForkExecutor for task cancellation support, in order to enforce timeouts. Bug: https://bugs.gentoo.org/654390 pym/portage/sync/modules/rsync/rsync.py | 14 ++ 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py index 070798a53..382a1eaae 100644 --- a/pym/portage/sync/modules/rsync/rsync.py +++ b/pym/portage/sync/modules/rsync/rsync.py @@ -26,6 +26,7 @@ from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM from portage.util._eventloop.global_event_loop import global_event_loop from portage.util import writemsg, writemsg_stdout from portage.util.futures import asyncio +from portage.util.futures.executor.fork import ForkExecutor from portage.sync.getaddrinfo_validate import getaddrinfo_validate from _emerge.UserQuery import UserQuery from portage.sync.syncbase import NewBase @@ -170,11 +171,16 @@ class RsyncSync(NewBase): level=logging.ERROR, noiselevel=-1) raise # retry + # The ThreadPoolExecutor that asyncio uses by default + # does not support cancellation of tasks, therefore + # use ForkExecutor for task cancellation support, in + # order to enforce timeouts. loop = global_event_loop() - func_coroutine = functools.partial(loop.run_in_executor, - None, noisy_refresh_keys) - decorated_func = retry_decorator(func_coroutine, loop=loop) - loop.run_until_complete(decorated_func()) + with ForkExecutor(loop=loop) as executor: + func_coroutine = functools.partial(loop.run_in_executor, + executor, noisy_refresh_keys) + decorated_func = retry_decorator(func_coroutine, loop=loop) + loop.run_until_complete(decorated_func()) out.eend(0) except (GematoException, asyncio.TimeoutError) as e: writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n"
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/, pym/portage/tests/util/futures/, ...
commit: b3b15c451cc21a2c53638f0eacc8396e395dcab3 Author: Zac Medico gentoo org> AuthorDate: Mon May 7 06:24:22 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon May 7 06:28:47 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b3b15c45 retry: add loop parameter during decoration pym/portage/sync/modules/rsync/rsync.py | 2 +- pym/portage/tests/util/futures/test_retry.py | 16 pym/portage/util/futures/retry.py| 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py index 1b8941ff6..070798a53 100644 --- a/pym/portage/sync/modules/rsync/rsync.py +++ b/pym/portage/sync/modules/rsync/rsync.py @@ -173,7 +173,7 @@ class RsyncSync(NewBase): loop = global_event_loop() func_coroutine = functools.partial(loop.run_in_executor, None, noisy_refresh_keys) - decorated_func = retry_decorator(func_coroutine) + decorated_func = retry_decorator(func_coroutine, loop=loop) loop.run_until_complete(decorated_func()) out.eend(0) except (GematoException, asyncio.TimeoutError) as e: diff --git a/pym/portage/tests/util/futures/test_retry.py b/pym/portage/tests/util/futures/test_retry.py index 16ecccbc7..7a1e76280 100644 --- a/pym/portage/tests/util/futures/test_retry.py +++ b/pym/portage/tests/util/futures/test_retry.py @@ -85,7 +85,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wrap_coroutine_func(SucceedLater(1)) decorator = retry(try_max=, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine) + decorated_func = decorator(func_coroutine, loop=loop) result = loop.run_until_complete(decorated_func()) self.assertEqual(result, 'success') @@ -94,7 +94,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wrap_coroutine_func(SucceedNever()) decorator = retry(try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine) + decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception().__cause__, SucceedNeverException)) @@ -104,7 +104,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wrap_coroutine_func(SucceedNever()) decorator = retry(reraise=True, try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine) + decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception(), SucceedNeverException)) @@ -114,7 +114,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wrap_coroutine_func(HangForever()) decorator = retry(try_max=2, try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine) + decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception().__cause__, asyncio.TimeoutError)) @@ -124,7 +124,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wrap_coroutine_func(HangForever()) decorator = retry(reraise=True, try_max=2, try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine) + decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception(), asyncio.TimeoutError)) @@ -134,7 +134,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wr
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/
commit: 299a9a40536209c4c2664857b5aecb018fa0a6f6 Author: Zac Medico gentoo org> AuthorDate: Sun May 6 23:40:33 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon May 7 00:19:54 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=299a9a40 TestCase: handle SkipTest during setUp method pym/portage/tests/__init__.py | 20 ++-- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pym/portage/tests/__init__.py b/pym/portage/tests/__init__.py index bf681c47e..e149b5c0c 100644 --- a/pym/portage/tests/__init__.py +++ b/pym/portage/tests/__init__.py @@ -207,18 +207,18 @@ class TestCase(unittest.TestCase): result.startTest(self) testMethod = getattr(self, self._testMethodName) try: - try: - self.setUp() - except SystemExit: - raise - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - return - ok = False try: + try: + self.setUp() + except KeyboardInterrupt: + raise + except SkipTest: + raise + except Exception: + result.addError(self, sys.exc_info()) + return + testMethod() ok = True except SkipTest as e:
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/
commit: fefc6f1feb57503ce4826de1e405c12ef761c0ce Author: Zac Medico gentoo org> AuthorDate: Sun May 6 23:41:15 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon May 7 00:19:54 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fefc6f1f RetryTestCase: test ThreadPoolExecutor and ForkExecutor (bug 654390) ThreadPoolExecutor is the default asyncio event loop's default executor, so explicitly test it. Also explicitly test ForkExecutor, since it is more useful in cases where tasks may need to be forcefully cancelled. Bug: https://bugs.gentoo.org/654390 pym/portage/tests/util/futures/test_retry.py | 37 ++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/pym/portage/tests/util/futures/test_retry.py b/pym/portage/tests/util/futures/test_retry.py index baf293d56..16ecccbc7 100644 --- a/pym/portage/tests/util/futures/test_retry.py +++ b/pym/portage/tests/util/futures/test_retry.py @@ -1,16 +1,24 @@ # Copyright 2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +try: + from concurrent.futures import ThreadPoolExecutor +except ImportError: + ThreadPoolExecutor = None + try: import threading except ImportError: import dummy_threading as threading +import sys + from portage.tests import TestCase from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.backoff import RandomExponentialBackoff from portage.util.futures import asyncio from portage.util.futures.retry import retry +from portage.util.futures.executor.fork import ForkExecutor from portage.util.monotonic import monotonic @@ -155,13 +163,31 @@ class RetryTestCase(TestCase): self.assertTrue(isinstance(done.pop().exception().__cause__, asyncio.TimeoutError)) -class RetryExecutorTestCase(RetryTestCase): +class RetryForkExecutorTestCase(RetryTestCase): """ Wrap each coroutine function with AbstractEventLoop.run_in_executor, in order to test the event loop's default executor. The executor may use either a thread or a subprocess, and either case is automatically detected and handled. """ + def __init__(self, *pargs, **kwargs): + super(RetryForkExecutorTestCase, self).__init__(*pargs, **kwargs) + self._executor = None + + def _setUpExecutor(self): + self._executor = ForkExecutor() + + def _tearDownExecutor(self): + if self._executor is not None: + self._executor.shutdown(wait=True) + self._executor = None + + def setUp(self): + self._setUpExecutor() + + def tearDown(self): + self._tearDownExecutor() + def _wrap_coroutine_func(self, coroutine_func): parent_loop = global_event_loop() @@ -191,7 +217,7 @@ class RetryExecutorTestCase(RetryTestCase): def execute_wrapper(): kill_switch = parent_loop.create_future() parent_future = asyncio.ensure_future( - parent_loop.run_in_executor(None, wrapper, kill_switch), + parent_loop.run_in_executor(self._executor, wrapper, kill_switch), loop=parent_loop) parent_future.add_done_callback( lambda parent_future: None if kill_switch.done() @@ -199,3 +225,10 @@ class RetryExecutorTestCase(RetryTestCase): return parent_future return execute_wrapper + + +class RetryThreadExecutorTestCase(RetryForkExecutorTestCase): + def _setUpExecutor(self): + if sys.version_info.major < 3: + self.skipTest('ThreadPoolExecutor not supported for python2') + self._executor = ThreadPoolExecutor(max_workers=1)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: a1360e017b7ee8156ad4ad850e2f8ea40228ca1a Author: Zac Medico gentoo org> AuthorDate: Mon May 7 00:06:07 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon May 7 00:19:54 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a1360e01 EventLoop.run_in_executor: use asyncio.wrap_future Since executors from the concurrent.futures package return concurrent.futures.Future, it's necessary to wrap them. pym/portage/util/_eventloop/EventLoop.py | 11 ++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index fc7380b03..de0795224 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -13,6 +13,11 @@ import signal import sys import traceback +try: + import asyncio as _real_asyncio +except ImportError: + _real_asyncio = None + try: import fcntl except ImportError: @@ -937,7 +942,11 @@ class EventLoop(object): if executor is None: executor = ForkExecutor(loop=self) self._default_executor = executor - return executor.submit(func, *args) + future = executor.submit(func, *args) + if _real_asyncio is not None: + future = _real_asyncio.wrap_future(future, + loop=self._asyncio_wrapper) + return future def is_running(self): """Return whether the event loop is currently running."""
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/_asyncio/, pym/portage/tests/util/futures/asyncio/, ...
commit: 85ac23b7c0c58cef72d22281d66d086521c01e3e Author: Zac Medico gentoo org> AuthorDate: Sun May 6 11:05:03 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun May 6 11:41:45 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=85ac23b7 asyncio: add _wrap_loop helper (bug 654390) In order to deal with asyncio event loop compatibility issues, add a _wrap_loop helper. For example, since python3.4 does not have the AbstractEventLoop.create_future() method, this helper function can be used to add a wrapper that implements the create_future method for python3.4. Bug: https://bugs.gentoo.org/654390 pym/portage/dbapi/porttree.py | 12 ++-- .../tests/util/futures/asyncio/test_child_watcher.py | 2 +- .../util/futures/asyncio/test_event_loop_in_fork.py | 8 .../tests/util/futures/asyncio/test_pipe_closed.py| 4 ++-- .../util/futures/asyncio/test_run_until_complete.py | 2 +- .../util/futures/asyncio/test_subprocess_exec.py | 4 ++-- pym/portage/util/futures/_asyncio/__init__.py | 19 ++- pym/portage/util/futures/_asyncio/tasks.py| 7 +-- pym/portage/util/futures/executor/fork.py | 4 ++-- pym/portage/util/futures/iter_completed.py| 7 +++ pym/portage/util/futures/retry.py | 3 +-- 11 files changed, 45 insertions(+), 27 deletions(-) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 801b5658a..3e36024ff 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -36,7 +36,7 @@ from portage import _encodings from portage import _unicode_encode from portage import OrderedDict from portage.util._eventloop.EventLoop import EventLoop -from portage.util._eventloop.global_event_loop import global_event_loop +from portage.util.futures import asyncio from portage.util.futures.iter_completed import iter_gather from _emerge.EbuildMetadataPhase import EbuildMetadataPhase @@ -325,8 +325,8 @@ class portdbapi(dbapi): @property def _event_loop(self): if portage._internal_caller: - # For internal portage usage, the global_event_loop is safe. - return global_event_loop() + # For internal portage usage, asyncio._wrap_loop() is safe. + return asyncio._wrap_loop() else: # For external API consumers, use a local EventLoop, since # we don't want to assume that it's safe to override the @@ -611,7 +611,7 @@ class portdbapi(dbapi): # to simultaneous instantiation of multiple event loops here. # Callers of this method certainly want the same event loop to # be used for all calls. - loop = loop or global_event_loop() + loop = asyncio._wrap_loop(loop) future = loop.create_future() cache_me = False if myrepo is not None: @@ -751,7 +751,7 @@ class portdbapi(dbapi): a set of alternative URIs. @rtype: asyncio.Future (or compatible) """ - loop = loop or global_event_loop() + loop = asyncio._wrap_loop(loop) result = loop.create_future() def aux_get_done(aux_get_future): @@ -1419,7 +1419,7 @@ def _async_manifest_fetchlist(portdb, repo_config, cp, cpv_list=None, @return: a Future resulting in a Mapping compatible with FetchlistDict @rtype: asyncio.Future (or compatible) """ - loop = loop or global_event_loop() + loop = asyncio._wrap_loop(loop) result = loop.create_future() cpv_list = (portdb.cp_list(cp, mytree=repo_config.location) if cpv_list is None else cpv_list) diff --git a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py index dca01be56..8ef497544 100644 --- a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py +++ b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py @@ -28,7 +28,7 @@ class ChildWatcherTestCase(TestCase): args_tuple = ('hello', 'world') - loop = asyncio.get_event_loop() + loop = asyncio._wrap_loop() future = loop.create_future() def callback(pid, returncode, *args): diff --git a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py index 7868d792a..19588bf3a 100644 --- a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py +++ b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py @@ -11,14 +11,14 @@ from portage.util.futures.unix_events import DefaultEventL
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/
commit: 87aeab1a62cc6fa1d48354a42ec4fa787dbe9603 Author: Zac Medico gentoo org> AuthorDate: Sun May 6 01:19:08 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun May 6 01:26:50 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=87aeab1a WriterPipeClosedTestCase: retry filling pipe This should suppress spurious writer callback observed twice for pypy in travis. See: https://travis-ci.org/gentoo/portage/jobs/375411936 See: https://travis-ci.org/gentoo/portage/jobs/373734825 .../tests/util/futures/asyncio/test_pipe_closed.py | 39 +- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py index 5398ca35c..e63829888 100644 --- a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py +++ b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py @@ -105,25 +105,32 @@ class WriterPipeClosedTestCase(_PipeClosedTestCase, TestCase): writer_callback.called = loop.create_future() _set_nonblocking(write_end.fileno()) + loop.add_writer(write_end.fileno(), writer_callback) - # Fill up the pipe, so that no writer callbacks should be - # received until the state has changed. - while True: - try: - os.write(write_end.fileno(), 512 * b'0') - except EnvironmentError as e: - if e.errno != errno.EAGAIN: - raise + # With pypy we've seen intermittent spurious writer callbacks + # here, so retry until the correct state is achieved. + tries = 10 + while tries: + tries -= 1 + + # Fill up the pipe, so that no writer callbacks should be + # received until the state has changed. + while True: + try: + os.write(write_end.fileno(), 512 * b'0') + except EnvironmentError as e: + if e.errno != errno.EAGAIN: + raise + break + + # Allow the loop to check for IO events, and assert + # that our future is still not done. + loop.run_until_complete(asyncio.sleep(0, loop=loop)) + if writer_callback.called.done(): + writer_callback.called = loop.create_future() + else: break - # We've seen at least one spurious writer callback when - # this was registered before the pipe was filled, so - # register it afterwards. - loop.add_writer(write_end.fileno(), writer_callback) - - # Allow the loop to check for IO events, and assert - # that our future is still not done. - loop.run_until_complete(asyncio.sleep(0, loop=loop)) self.assertFalse(writer_callback.called.done()) # Demonstrate that the callback is called afer the
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/
commit: 5a5ed99cb5a6e8913df2e9ca29b4b4d5c179c20f Author: Zac Medico gentoo org> AuthorDate: Sat May 5 23:04:10 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun May 6 00:35:44 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5a5ed99c RetryTestCase: support ThreadPoolExecutor (bug 654390) In order to support the default asyncio event loop's ThreadPoolExecutor, use a threading.Event instance to support cancellation of tasks. Bug: https://bugs.gentoo.org/654390 pym/portage/tests/util/futures/test_retry.py | 96 +--- 1 file changed, 74 insertions(+), 22 deletions(-) diff --git a/pym/portage/tests/util/futures/test_retry.py b/pym/portage/tests/util/futures/test_retry.py index cdca7d294..781eac9a1 100644 --- a/pym/portage/tests/util/futures/test_retry.py +++ b/pym/portage/tests/util/futures/test_retry.py @@ -1,8 +1,6 @@ # Copyright 2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import functools - try: import threading except ImportError: @@ -28,10 +26,17 @@ class SucceedLater(object): self._succeed_time = monotonic() + duration def __call__(self): + loop = global_event_loop() + result = loop.create_future() remaining = self._succeed_time - monotonic() if remaining > 0: - raise SucceedLaterException('time until success: {} seconds'.format(remaining)) - return 'success' + loop.call_soon_threadsafe(lambda: None if result.done() else + result.set_exception(SucceedLaterException( + 'time until success: {} seconds'.format(remaining + else: + loop.call_soon_threadsafe(lambda: None if result.done() else + result.set_result('success')) + return result class SucceedNeverException(Exception): @@ -43,7 +48,11 @@ class SucceedNever(object): A callable object that never succeeds. """ def __call__(self): - raise SucceedNeverException('expected failure') + loop = global_event_loop() + result = loop.create_future() + loop.call_soon_threadsafe(lambda: None if result.done() else + result.set_exception(SucceedNeverException('expected failure'))) + return result class HangForever(object): @@ -51,14 +60,21 @@ class HangForever(object): A callable object that sleeps forever. """ def __call__(self): - threading.Event().wait() + return global_event_loop().create_future() class RetryTestCase(TestCase): + + def _wrap_coroutine_func(self, coroutine_func): + """ + Derived classes may override this method in order to implement + alternative forms of execution. + """ + return coroutine_func + def testSucceedLater(self): loop = global_event_loop() - func = SucceedLater(1) - func_coroutine = functools.partial(loop.run_in_executor, None, func) + func_coroutine = self._wrap_coroutine_func(SucceedLater(1)) decorator = retry(try_max=, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) @@ -67,8 +83,7 @@ class RetryTestCase(TestCase): def testSucceedNever(self): loop = global_event_loop() - func = SucceedNever() - func_coroutine = functools.partial(loop.run_in_executor, None, func) + func_coroutine = self._wrap_coroutine_func(SucceedNever()) decorator = retry(try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) @@ -78,8 +93,7 @@ class RetryTestCase(TestCase): def testSucceedNeverReraise(self): loop = global_event_loop() - func = SucceedNever() - func_coroutine = functools.partial(loop.run_in_executor, None, func) + func_coroutine = self._wrap_coroutine_func(SucceedNever()) decorator = retry(reraise=True, try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) @@ -89,8 +103,7 @@ class RetryTestCase(TestCase): def testHangForever(self): loop = global_event_loop() - func = HangForever() - func_coroutine = functools.partial(loop.run_in_executor, None, func) + func_coroutine = self._wrap_coroutine_func(HangForever())
[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/, pym/portage/util/_async/, ...
commit: 82a3cda6f1ffbbbd75c962d610c7eb4a04a865d9 Author: Zac Medico gentoo org> AuthorDate: Sat May 5 21:00:51 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat May 5 21:00:51 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=82a3cda6 Minimize _asyncio_wrapper usage (bug 654390) Since migration to asyncio compatible APIs is now complete, minimize _asyncio_wrapper usage. Also remove _asyncio_wrapper from SchedulerInterface, since there are no more consumers. Bug: https://bugs.gentoo.org/654390 pym/_emerge/AbstractPollTask.py | 2 +- pym/portage/_emirrordist/FetchIterator.py| 1 - pym/portage/dbapi/porttree.py| 3 --- .../tests/util/futures/asyncio/test_subprocess_exec.py | 1 - pym/portage/tests/util/futures/test_iter_completed.py| 4 ++-- pym/portage/tests/util/futures/test_retry.py | 16 pym/portage/util/_async/SchedulerInterface.py| 1 - pym/portage/util/futures/__init__.py | 5 + pym/portage/util/futures/_asyncio/tasks.py | 1 - pym/portage/util/futures/executor/fork.py| 3 +-- pym/portage/util/futures/futures.py | 2 +- pym/portage/util/futures/iter_completed.py | 3 --- 12 files changed, 14 insertions(+), 28 deletions(-) diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py index dff4b3efd..f898aa708 100644 --- a/pym/_emerge/AbstractPollTask.py +++ b/pym/_emerge/AbstractPollTask.py @@ -109,7 +109,7 @@ class AbstractPollTask(AsynchronousTask): self._registered = False def _wait_loop(self, timeout=None): - loop = getattr(self.scheduler, '_asyncio_wrapper', self.scheduler) + loop = self.scheduler tasks = [self.async_wait()] if timeout is not None: tasks.append(asyncio.ensure_future( diff --git a/pym/portage/_emirrordist/FetchIterator.py b/pym/portage/_emirrordist/FetchIterator.py index 366453c12..04d4da62b 100644 --- a/pym/portage/_emirrordist/FetchIterator.py +++ b/pym/portage/_emirrordist/FetchIterator.py @@ -125,7 +125,6 @@ def _async_fetch_tasks(config, hash_filter, repo_config, digests_future, cpv, instances for each of the files referenced by an ebuild. @rtype: asyncio.Future (or compatible) """ - loop = getattr(loop, '_asyncio_wrapper', loop) result = loop.create_future() fetch_tasks = [] diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 6c38232bb..801b5658a 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -612,7 +612,6 @@ class portdbapi(dbapi): # Callers of this method certainly want the same event loop to # be used for all calls. loop = loop or global_event_loop() - loop = getattr(loop, '_asyncio_wrapper', loop) future = loop.create_future() cache_me = False if myrepo is not None: @@ -753,7 +752,6 @@ class portdbapi(dbapi): @rtype: asyncio.Future (or compatible) """ loop = loop or global_event_loop() - loop = getattr(loop, '_asyncio_wrapper', loop) result = loop.create_future() def aux_get_done(aux_get_future): @@ -1422,7 +1420,6 @@ def _async_manifest_fetchlist(portdb, repo_config, cp, cpv_list=None, @rtype: asyncio.Future (or compatible) """ loop = loop or global_event_loop() - loop = getattr(loop, '_asyncio_wrapper', loop) result = loop.create_future() cpv_list = (portdb.cp_list(cp, mytree=repo_config.location) if cpv_list is None else cpv_list) diff --git a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py index be103a9e0..98983941d 100644 --- a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py +++ b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py @@ -24,7 +24,6 @@ def reader(input_file, loop=None): @rtype: asyncio.Future (or compatible) """ loop = loop or asyncio.get_event_loop() - loop = getattr(loop, '_asyncio_wrapper', loop) future = loop.create_future() _Reader(future, input_file, loop) return future diff --git a/pym/portage/tests/util/futures/test_iter_completed.py b/pym/portage/tests/util/futures/test_iter_completed.py index 90668eb02..9ab410a9e 100644 --- a/pym/portage/tests/util/futures/test_iter_completed.py +++ b/pym/portage/tests/util/futures/test_iter_completed.py @@ -34,7 +34,7 @@ class IterCompletedTestCase(TestCase): # load causes the tasks to finish in an unexpected order. self.todo = True -
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/resolver/, pym/_emerge/
commit: ce150da22e351a7ba52a6390b9cb7aa076c0c8ce Author: Zac Medico gentoo org> AuthorDate: Fri May 4 03:21:40 2018 + Commit: Zac Medico gentoo org> CommitDate: Fri May 4 17:11:07 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ce150da2 depgraph.autounmask_breakage_detected: ignore REQUIRED_USE violations (bug 654782) When autounmask USE changes violate REQUIRED_USE, rather than recalculate with autounmask disabled, display the autounmask USE changes along with the REQUIRED_USE violation. Adjust unit tests to allow for the autounmask USE changes that were previously discarded. For the case reported in bug 654782, the output will appear as follows: The following keyword changes are necessary to proceed: (see "package.accept_keywords" in the portage(5) man page for more details) # required by =retext-7.0.1-r1 (argument) =app-editors/retext-7.0.1-r1 ~amd64 The following USE changes are necessary to proceed: (see "package.use" in the portage(5) man page for more details) # required by app-editors/retext-7.0.1-r1::gentoo # required by =retext-7.0.1-r1 (argument) >=dev-python/PyQt5-5.9.2 printsupport webengine network gui widgets !!! The ebuild selected to satisfy "dev-python/PyQt5[gui,network,printsupport,webengine,widgets,python_targets_python3_4(-)?,python_targets_python3_5(-)?,python_targets_python3_6(-)?,-python_single_target_python3_4(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-)]" has unmet requirements. - dev-python/PyQt5-5.9.2::gentoo USE="-bluetooth -dbus -debug -declarative -designer -examples -gles2 -gui -help -location -multimedia -network -opengl -positioning -printsupport -sensors -serialport -sql -svg -testlib -webchannel -webengine -webkit -websockets -widgets -x11extras -xmlpatterns" PYTHON_TARGETS="python2_7 python3_4 python3_6 -python3_5" The following REQUIRED_USE flag constraints are unsatisfied: webengine? ( widgets? ( webchannel ) ) The above constraints are a subset of the following complete expression: any-of ( python_targets_python2_7 python_targets_python3_4 python_targets_python3_5 python_targets_python3_6 ) bluetooth? ( gui ) declarative? ( gui network ) designer? ( widgets ) help? ( gui widgets ) location? ( positioning ) multimedia? ( gui network ) opengl? ( gui widgets ) positioning? ( gui ) printsupport? ( gui widgets ) sensors? ( gui ) serialport? ( gui ) sql? ( widgets ) svg? ( gui widgets ) testlib? ( widgets ) webchannel? ( network ) webengine? ( network widgets? ( printsupport webchannel ) ) webkit? ( gui network printsupport widgets ) websockets? ( network ) widgets? ( gui ) xmlpatterns? ( network ) (dependency required by "app-editors/retext-7.0.1-r1::gentoo" [ebuild]) (dependency required by "=retext-7.0.1-r1" [argument]) Bug: https://bugs.gentoo.org/654782 Reviewed-by: M. J. Everitt iee.org> pym/_emerge/depgraph.py| 8 ++-- pym/portage/tests/resolver/test_autounmask.py | 48 -- pym/portage/tests/resolver/test_slot_collisions.py | 20 + 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index fbd16ad98..429d8871c 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -3009,6 +3009,10 @@ class depgraph(object): {"myparent" : dep.parent, "show_req_use" : pkg})) self._dynamic_config._required_use_unsatisfied = True self._dynamic_config._skip_restart = True + # Add pkg to digraph in order to enable autounmask messages + # for this package, which is useful when autounmask USE + # changes have violated REQUIRED_USE. + self._dynamic_config.digraph.add(pkg, dep.parent, priority=priority) return 0 if not pkg.onlydeps: @@ -9428,10 +9432,6 @@ class depgraph(object): return self._dynamic_config._need_config_reload def autounmask_breakage_detected(self): - # Check for REQUIRED_USE violations. - for changes in self._dynamic_config._needed_use_config_changes.values(): - if getattr(changes, 'required_use_satisfied', None) is False: - return True try: for pargs, kwargs in self._dynamic_config._unsatisfied_deps_for_display: self._show_unsatisfied_dep( diff --git a/pym/portage/tests/resolver/test_autounmask.py b/pym/portage/tests/resolver/test_autounmask.py index 9042349ea..809d42104 100644 --- a/pym/portage/tests/resolver/test_autounmask.py +++ b/pym/portage/tests/resolver/test_autounmask.py @@ -251,15 +251,42 @@ class AutounmaskTestCase(TestCase):
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/_config/
commit: b69a756099973abbf9719717ea3726519b32ce60 Author: Zac Medico gentoo org> AuthorDate: Thu May 3 23:00:35 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu May 3 23:08:47 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b69a7560 LocationsManager: fix SYSROOT normalization to handle empty SYSROOT This issue was exposed when building portage-2.3.34 with USE="epydoc" enabled, since epydoc triggers instantiation of portage.settings. The empty SYSROOT change for bug 654600 (in commit 1b5110557d1dd725f7c12bbed4b7ceaaec29f2a3) triggered incorrect normalization behavior here. Fixes: a41dacf7926c ("Export SYSROOT and ESYSROOT in ebuild env in EAPI 7") pym/portage/package/ebuild/_config/LocationsManager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pym/portage/package/ebuild/_config/LocationsManager.py b/pym/portage/package/ebuild/_config/LocationsManager.py index b57443ba7..f7d7209ff 100644 --- a/pym/portage/package/ebuild/_config/LocationsManager.py +++ b/pym/portage/package/ebuild/_config/LocationsManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2014 Gentoo Foundation +# Copyright 2010-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -69,7 +69,7 @@ class LocationsManager(object): if self.sysroot is None: self.sysroot = "/" else: - self.sysroot = normalize_path(os.path.abspath(self.sysroot)).rstrip(os.sep) + os.sep + self.sysroot = normalize_path(os.path.abspath(self.sysroot or os.sep)).rstrip(os.sep) + os.sep self.esysroot = self.sysroot.rstrip(os.sep) + self.eprefix + os.sep
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/_config/
commit: cd7364fbbb048564c2714d8a08df1ecbdc9cfe6e Author: Zac Medico gentoo org> AuthorDate: Thu May 3 20:27:59 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu May 3 20:33:27 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cd7364fb special_env_vars: add ESYSROOT and SYSROOT to environ_whitelist Since these variables are filtered from the peristent ebuild environment by the __filter_readonly_variables function, they have to be whitelisted so that the config.environ() method allow them to pass through. Fixes: a41dacf7926c ("Export SYSROOT and ESYSROOT in ebuild env in EAPI 7") Reported-by: James Le Cuirot gentoo.org> pym/portage/package/ebuild/_config/special_env_vars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pym/portage/package/ebuild/_config/special_env_vars.py b/pym/portage/package/ebuild/_config/special_env_vars.py index 3323ff064..a308518af 100644 --- a/pym/portage/package/ebuild/_config/special_env_vars.py +++ b/pym/portage/package/ebuild/_config/special_env_vars.py @@ -46,7 +46,7 @@ environ_whitelist += [ "DISTDIR", "DOC_SYMLINKS_DIR", "EAPI", "EBUILD", "EBUILD_FORCE_TEST", "EBUILD_PHASE", "EBUILD_PHASE_FUNC", "ECLASSDIR", "ECLASS_DEPTH", "ED", - "EMERGE_FROM", "EPREFIX", "EROOT", + "EMERGE_FROM", "EPREFIX", "EROOT", "ESYSROOT", "FEATURES", "FILESDIR", "HOME", "MERGE_TYPE", "NOCOLOR", "PATH", "PKGDIR", "PKGUSE", "PKG_LOGDIR", "PKG_TMPDIR", @@ -77,7 +77,7 @@ environ_whitelist += [ "PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", "PORTAGE_XATTR_EXCLUDE", "PORTDIR", "PORTDIR_OVERLAY", "PREROOTPATH", "PYTHONDONTWRITEBYTECODE", "REPLACING_VERSIONS", "REPLACED_BY_VERSION", - "ROOT", "ROOTPATH", "T", "TMP", "TMPDIR", + "ROOT", "ROOTPATH", "SYSROOT", "T", "TMP", "TMPDIR", "USE_EXPAND", "USE_ORDER", "WORKDIR", "XARGS", "__PORTAGE_TEST_HARDLINK_LOCKS", ]
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/
commit: 1b5110557d1dd725f7c12bbed4b7ceaaec29f2a3 Author: Zac Medico gentoo org> AuthorDate: Thu May 3 00:55:48 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu May 3 18:45:07 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1b511055 config.environ: always strip slash from SYSROOT (bug 654600) Since SYSROOT=/ interacts badly with autotools.eclass (bug 654600), and no EAPI expects SYSROOT to have a trailing slash, always strip the trailing slash from SYSROOT. Bug: https://bugs.gentoo.org/654600 Fixes: a41dacf7926c ("Export SYSROOT and ESYSROOT in ebuild env in EAPI 7") Reviewed-by: James Le Cuirot gentoo.org> Reviewed-by: Michał Górny gentoo.org> Reviewed-by: Brian Dolbec gentoo.org> pym/portage/package/ebuild/config.py | 9 +++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index f9b257b86..88acac5cc 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -2813,11 +2813,16 @@ class config(object): mydict.pop("ECLASSDIR", None) if not eapi_attrs.path_variables_end_with_trailing_slash: - for v in ("D", "ED", "ROOT", "EROOT", "SYSROOT", "ESYSROOT", - "BROOT"): + for v in ("D", "ED", "ROOT", "EROOT", "ESYSROOT", "BROOT"): if v in mydict: mydict[v] = mydict[v].rstrip(os.path.sep) + # Since SYSROOT=/ interacts badly with autotools.eclass (bug 654600), + # and no EAPI expects SYSROOT to have a trailing slash, always strip + # the trailing slash from SYSROOT. + if 'SYSROOT' in mydict: + mydict['SYSROOT'] = mydict['SYSROOT'].rstrip(os.sep) + try: builddir = mydict["PORTAGE_BUILDDIR"] distdir = mydict["DISTDIR"]
[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/
commit: 35ca4ac2f8f0de61289bf812681c21d14d2524db Author: Zac Medico gentoo org> AuthorDate: Thu May 3 06:26:19 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu May 3 06:35:45 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=35ca4ac2 rsync: silence signature info for --quiet mode (bug 654664) Silence messages like the following for --quiet mode: * Using keys from /var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg * Refreshing keys from keyserver ...[ ok ] * Manifest timestamp: 2018-05-03 05:38:39 UTC * Valid OpenPGP signature found: * - primary key: DCD05B71EAB94199527F44ACDB6B8C1F96D8BF6D * - subkey: E1D6ABB63BFCFB4BA02FDF1CEC590EEAC9189250 * - timestamp: 2018-05-03 05:38:39 UTC * Verifying /usr/portage ...[ ok ] Bug: https://bugs.gentoo.org/654664 pym/portage/sync/modules/rsync/rsync.py | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py index b2850715e..1b8941ff6 100644 --- a/pym/portage/sync/modules/rsync/rsync.py +++ b/pym/portage/sync/modules/rsync/rsync.py @@ -66,7 +66,8 @@ class RsyncSync(NewBase): opts = self.options.get('emerge_config').opts self.usersync_uid = self.options.get('usersync_uid', None) enter_invalid = '--ask-enter-invalid' in opts - out = portage.output.EOutput() + quiet = '--quiet' in opts + out = portage.output.EOutput(quiet=quiet) syncuri = self.repo.sync_uri if self.repo.module_specific_options.get( 'sync-rsync-vcs-ignore', 'false').lower() == 'true': @@ -385,10 +386,12 @@ class RsyncSync(NewBase): raise RuntimeError('Timestamp not found in Manifest') if (self.max_age != 0 and (datetime.datetime.utcnow() - ts.ts).days > self.max_age): + out.quiet = False out.ewarn('Manifest is over %d days old, this is suspicious!' % (self.max_age,)) out.ewarn('You may want to try using another mirror and/or reporting this one:') out.ewarn(' %s' % (dosyncuri,)) out.ewarn('') + out.quiet = quiet out.einfo('Manifest timestamp: %s UTC' % (ts.ts,)) out.einfo('Valid OpenPGP signature found:')
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/
commit: 10253819aae4cefee80f377e167ad521516d66a2 Author: Zac Medico gentoo org> AuthorDate: Thu May 3 01:18:39 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu May 3 01:18:39 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=10253819 WriterPipeClosedTestCase: add_writer after pipe is filled Hopefully this supresses a spurious writer callback observed once for pypy in travis. See: https://travis-ci.org/gentoo/portage/jobs/373734825 pym/portage/tests/util/futures/asyncio/test_pipe_closed.py | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py index 1ecddab78..5398ca35c 100644 --- a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py +++ b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py @@ -105,7 +105,6 @@ class WriterPipeClosedTestCase(_PipeClosedTestCase, TestCase): writer_callback.called = loop.create_future() _set_nonblocking(write_end.fileno()) - loop.add_writer(write_end.fileno(), writer_callback) # Fill up the pipe, so that no writer callbacks should be # received until the state has changed. @@ -117,6 +116,11 @@ class WriterPipeClosedTestCase(_PipeClosedTestCase, TestCase): raise break + # We've seen at least one spurious writer callback when + # this was registered before the pipe was filled, so + # register it afterwards. + loop.add_writer(write_end.fileno(), writer_callback) + # Allow the loop to check for IO events, and assert # that our future is still not done. loop.run_until_complete(asyncio.sleep(0, loop=loop))
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/emerge/, bin/
commit: 4bda6c546aab816e835f62b326db9c2215e182ac Author: Zac Medico gentoo org> AuthorDate: Tue May 1 23:37:33 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue May 1 23:49:44 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4bda6c54 Revert "phase-helpers.sh: fix best/has_version -b for cross-prefix" This reverts commit a0ac6e6727abec8d2482c95b1e84d8df24d78619, since BROOT is only supposed to be set in src_* phases. Update SimpleEmergeTestCase to call best/has_version -b only in src_install for EAPI 7. Reported-by: James Le Cuirot gentoo.org> bin/phase-helpers.sh| 2 +- pym/portage/tests/emerge/test_simple.py | 23 --- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh index 8b16d7d31..59c19cf67 100644 --- a/bin/phase-helpers.sh +++ b/bin/phase-helpers.sh @@ -912,7 +912,7 @@ ___best_version_and_has_version_common() { case ${root_arg} in -r) root=${EROOT} ;; -d) root=${ESYSROOT} ;; - -b) root=${BROOT:-/${PORTAGE_OVERRIDE_EPREFIX#/}} ;; + -b) root=${BROOT:-/} ;; esac else case ${root_arg} in diff --git a/pym/portage/tests/emerge/test_simple.py b/pym/portage/tests/emerge/test_simple.py index 204c23296..b1402ddd5 100644 --- a/pym/portage/tests/emerge/test_simple.py +++ b/pym/portage/tests/emerge/test_simple.py @@ -58,6 +58,8 @@ src_install() { echo "blah blah blah" > "${T}"/latin-1-$(printf "\\xa9")-regular-file || die doins "${T}"/latin-1-$(printf "\\xa9")-regular-file dosym latin-1-$(printf "\\xa9")-regular-file ${latin_1_dir}/latin-1-$(printf "\\xa9")-symlink || die + + call_has_and_best_version } pkg_config() { @@ -69,14 +71,29 @@ pkg_info() { } pkg_preinst() { + if ! ___eapi_best_version_and_has_version_support_-b_-d_-r; then + # The BROOT variable is unset during pkg_* phases for EAPI 7, + # therefore best/has_version -b is expected to fail if we attempt + # to call it for EAPI 7 here. + call_has_and_best_version + fi +} + +call_has_and_best_version() { local root_arg if ___eapi_best_version_and_has_version_support_-b_-d_-r; then root_arg="-b" else root_arg="--host-root" fi - einfo "called pkg_preinst for $CATEGORY/$PF" - + einfo "called ${EBUILD_PHASE_FUNC} for $CATEGORY/$PF" + einfo "EPREFIX=${EPREFIX}" + einfo "PORTAGE_OVERRIDE_EPREFIX=${PORTAGE_OVERRIDE_EPREFIX}" + einfo "ROOT=${ROOT}" + einfo "EROOT=${EROOT}" + einfo "SYSROOT=${SYSROOT}" + einfo "ESYSROOT=${ESYSROOT}" + einfo "BROOT=${BROOT}" # Test that has_version and best_version work correctly with # prefix (involves internal ROOT -> EROOT calculation in order # to support ROOT override via the environment with EAPIs 3 @@ -90,7 +107,7 @@ pkg_preinst() { if [[ ${EPREFIX} != ${PORTAGE_OVERRIDE_EPREFIX} ]] ; then if has_version ${root_arg} $CATEGORY/$PN:$SLOT ; then einfo "has_version ${root_arg} detects an installed instance of $CATEGORY/$PN:$SLOT" - einfo "best_version ${root_arg} reports that the installed instance is $(best_version $CATEGORY/$PN:$SLOT)" + einfo "best_version ${root_arg} reports that the installed instance is $(best_version ${root_arg} $CATEGORY/$PN:$SLOT)" else einfo "has_version ${root_arg} does not detect an installed instance of $CATEGORY/$PN:$SLOT" fi
[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/
commit: d8b9d5d6250a4f9ebaf8db6cbab96a585c56aadd Author: Zac Medico gentoo org> AuthorDate: Tue May 1 19:52:28 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue May 1 19:52:28 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8b9d5d6 rsync: surface key refresh exceptions early (bug 649276) Since retry does not help for some types of errors, display errors as soon as they occur. Fixes: 5f29f03d0d97 ("rsync: add key refresh retry (bug 649276)") pym/portage/sync/modules/rsync/rsync.py | 14 +- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py index 180bf8571..b2850715e 100644 --- a/pym/portage/sync/modules/rsync/rsync.py +++ b/pym/portage/sync/modules/rsync/rsync.py @@ -157,9 +157,21 @@ class RsyncSync(NewBase): if retry_decorator is None: openpgp_env.refresh_keys() else: + def noisy_refresh_keys(): + """ + Since retry does not help for some types of + errors, display errors as soon as they occur. + """ + try: + openpgp_env.refresh_keys() + except Exception as e: + writemsg_level("%s\n" % (e,), + level=logging.ERROR, noiselevel=-1) + raise # retry + loop = global_event_loop() func_coroutine = functools.partial(loop.run_in_executor, - None, openpgp_env.refresh_keys) + None, noisy_refresh_keys) decorated_func = retry_decorator(func_coroutine) loop.run_until_complete(decorated_func()) out.eend(0)
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/emerge/, pym/portage/tests/resolver/, pym/portage/
commit: 75d6df1a2988ba440feed3db02550b62ebe0c204 Author: Zac Medico gentoo org> AuthorDate: Tue May 1 18:18:49 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue May 1 18:22:45 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=75d6df1a Enable EAPI 7 (bug 654472) Bug: https://bugs.gentoo.org/654472 pym/portage/__init__.py | 2 +- pym/portage/const.py| 2 +- pym/portage/tests/emerge/test_simple.py | 6 +++--- pym/portage/tests/resolver/test_eapi.py | 2 +- pym/portage/tests/resolver/test_required_use.py | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 15c404f6f..166bfc700 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -462,7 +462,7 @@ def abssymlink(symlink, target=None): _doebuild_manifest_exempt_depend = 0 -_testing_eapis = frozenset(["4-python", "4-slot-abi", "5-progress", "5-hdepend", "7_pre1"]) +_testing_eapis = frozenset(["4-python", "4-slot-abi", "5-progress", "5-hdepend", "7_pre1", "7"]) _deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1", "5_pre1", "5_pre2", "6_pre1"]) _supported_eapis = frozenset([str(x) for x in range(portage.const.EAPI + 1)] + list(_testing_eapis) + list(_deprecated_eapis)) diff --git a/pym/portage/const.py b/pym/portage/const.py index 16922a5e6..7f84bf0e9 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -204,7 +204,7 @@ SUPPORTED_FEATURES = frozenset([ "xattr", ]) -EAPI = 6 +EAPI = 7 HASHING_BLOCKSIZE= 32768 diff --git a/pym/portage/tests/emerge/test_simple.py b/pym/portage/tests/emerge/test_simple.py index 495e22297..204c23296 100644 --- a/pym/portage/tests/emerge/test_simple.py +++ b/pym/portage/tests/emerge/test_simple.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Gentoo Foundation +# Copyright 2011-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import subprocess @@ -116,13 +116,13 @@ pkg_preinst() { "MISC_CONTENT": install_something, }, "dev-libs/C-1": { - "EAPI" : "7_pre1", + "EAPI" : "7", "KEYWORDS": "~x86", "RDEPEND": "dev-libs/D[flag]", "MISC_CONTENT": install_something, }, "dev-libs/D-1": { - "EAPI" : "7_pre1", + "EAPI" : "7", "KEYWORDS": "~x86", "IUSE" : "flag", "MISC_CONTENT": install_something, diff --git a/pym/portage/tests/resolver/test_eapi.py b/pym/portage/tests/resolver/test_eapi.py index fce05890b..50b9d90da 100644 --- a/pym/portage/tests/resolver/test_eapi.py +++ b/pym/portage/tests/resolver/test_eapi.py @@ -62,7 +62,7 @@ class EAPITestCase(TestCase): #EAPI-7: implicit || ( ) no longer satisfies deps "dev-libs/C-1": { "EAPI": "6", "IUSE": "foo", "RDEPEND": "|| ( foo? ( dev-libs/B ) )" }, - "dev-libs/C-2": { "EAPI": "7_pre1", "IUSE": "foo", "RDEPEND": "|| ( foo? ( dev-libs/B ) )" }, + "dev-libs/C-2": { "EAPI": "7", "IUSE": "foo", "RDEPEND": "|| ( foo? ( dev-libs/B ) )" }, } test_cases = ( diff --git a/pym/portage/tests/resolver/test_required_use.py b/pym/portage/tests/resolver/test_required_use.py index 7909f927f..c679ce300 100644 --- a/pym/portage/tests/resolver/test_required_use.py +++ b/pym/portage/tests/resolver/test_required_use.py @@ -51,9 +51,9 @@ class RequiredUSETestCase(TestCase): "dev-libs/E-4" : {"EAPI": "5", "IUSE": "+foo +bar", "REQUIRED_USE": "?? ( foo bar )"}, "dev-libs/E-5" : {"EAPI": "5", "IUSE": "+foo +bar", "REQUIRED_USE": "?? ( )"}, - "dev-libs/F-1" : {"EAPI": "7_pre1", "IUSE": "+foo +bar", "REQUIRED_USE": "|| ( )"}, - "dev-libs/F-2" : {"EAPI": "7_pre1", "IUSE": "+foo +bar", "REQUIRED_USE": "^^ ( )"}, - "dev-libs/F-3" : {"EAPI": "7_pre1", "IUSE": "+foo +bar", "REQUIRED_USE": "?? ( )"}, + "dev-libs/F-1" : {"EAPI": "7", "IUSE": "+foo +bar", "REQUIRED_USE": "|| ( )"}, + "dev-libs/F-2" : {"EAPI": "7", "IUSE": "+foo +bar", "REQUIRED_USE": "^^ ( )"}, + "dev-libs/F-3" : {"EAPI": "7", "IUSE": "+foo +bar", "REQUIRED_USE": "?? ( )"}, } test_cases = (
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/emerge/, bin/
commit: a0ac6e6727abec8d2482c95b1e84d8df24d78619 Author: Zac Medico gentoo org> AuthorDate: Tue May 1 18:14:07 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue May 1 18:15:47 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a0ac6e67 phase-helpers.sh: fix best/has_version -b for cross-prefix Fixes: 43b6be7423aa ("phase-helpers.sh: Implement -r|-d|-b options for best/has_version") bin/phase-helpers.sh| 2 +- pym/portage/tests/emerge/test_simple.py | 12 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh index 59c19cf67..8b16d7d31 100644 --- a/bin/phase-helpers.sh +++ b/bin/phase-helpers.sh @@ -912,7 +912,7 @@ ___best_version_and_has_version_common() { case ${root_arg} in -r) root=${EROOT} ;; -d) root=${ESYSROOT} ;; - -b) root=${BROOT:-/} ;; + -b) root=${BROOT:-/${PORTAGE_OVERRIDE_EPREFIX#/}} ;; esac else case ${root_arg} in diff --git a/pym/portage/tests/emerge/test_simple.py b/pym/portage/tests/emerge/test_simple.py index 17dcd548d..495e22297 100644 --- a/pym/portage/tests/emerge/test_simple.py +++ b/pym/portage/tests/emerge/test_simple.py @@ -119,11 +119,13 @@ pkg_preinst() { "EAPI" : "7_pre1", "KEYWORDS": "~x86", "RDEPEND": "dev-libs/D[flag]", + "MISC_CONTENT": install_something, }, "dev-libs/D-1": { "EAPI" : "7_pre1", "KEYWORDS": "~x86", "IUSE" : "flag", + "MISC_CONTENT": install_something, }, "virtual/foo-0": { "EAPI" : "5", @@ -326,6 +328,16 @@ pkg_preinst() { portageq_cmd + ("match", eroot, "dev-libs/D[flag]"), # Test cross-prefix usage, including chpathtool for binpkgs. + # EAPI 7 + ({"EPREFIX" : cross_prefix},) + \ + emerge_cmd + ("dev-libs/C",), + ({"EPREFIX" : cross_prefix},) + \ + portageq_cmd + ("has_version", cross_prefix, "dev-libs/C"), + ({"EPREFIX" : cross_prefix},) + \ + portageq_cmd + ("has_version", cross_prefix, "dev-libs/D"), + ({"ROOT": cross_root},) + emerge_cmd + ("dev-libs/D",), + portageq_cmd + ("has_version", cross_eroot, "dev-libs/D"), + # EAPI 5 ({"EPREFIX" : cross_prefix},) + \ emerge_cmd + ("--usepkgonly", "dev-libs/A"), ({"EPREFIX" : cross_prefix},) + \
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/emerge/
commit: 844998daa50e9db0a337bc0cf9f665d224779665 Author: Zac Medico gentoo org> AuthorDate: Tue May 1 17:49:07 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue May 1 17:52:02 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=844998da SimpleEmergeTestCase: EAPI 7_pre1 has/best_version -b pym/portage/tests/emerge/test_simple.py | 18 -- 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pym/portage/tests/emerge/test_simple.py b/pym/portage/tests/emerge/test_simple.py index f99c77927..17dcd548d 100644 --- a/pym/portage/tests/emerge/test_simple.py +++ b/pym/portage/tests/emerge/test_simple.py @@ -69,6 +69,12 @@ pkg_info() { } pkg_preinst() { + local root_arg + if ___eapi_best_version_and_has_version_support_-b_-d_-r; then + root_arg="-b" + else + root_arg="--host-root" + fi einfo "called pkg_preinst for $CATEGORY/$PF" # Test that has_version and best_version work correctly with @@ -82,11 +88,11 @@ pkg_preinst() { einfo "has_version does not detect an installed instance of $CATEGORY/$PN:$SLOT" fi if [[ ${EPREFIX} != ${PORTAGE_OVERRIDE_EPREFIX} ]] ; then - if has_version --host-root $CATEGORY/$PN:$SLOT ; then - einfo "has_version --host-root detects an installed instance of $CATEGORY/$PN:$SLOT" - einfo "best_version --host-root reports that the installed instance is $(best_version $CATEGORY/$PN:$SLOT)" + if has_version ${root_arg} $CATEGORY/$PN:$SLOT ; then + einfo "has_version ${root_arg} detects an installed instance of $CATEGORY/$PN:$SLOT" + einfo "best_version ${root_arg} reports that the installed instance is $(best_version $CATEGORY/$PN:$SLOT)" else - einfo "has_version --host-root does not detect an installed instance of $CATEGORY/$PN:$SLOT" + einfo "has_version ${root_arg} does not detect an installed instance of $CATEGORY/$PN:$SLOT" fi fi } @@ -110,12 +116,12 @@ pkg_preinst() { "MISC_CONTENT": install_something, }, "dev-libs/C-1": { - "EAPI" : "6", + "EAPI" : "7_pre1", "KEYWORDS": "~x86", "RDEPEND": "dev-libs/D[flag]", }, "dev-libs/D-1": { - "EAPI" : "6", + "EAPI" : "7_pre1", "KEYWORDS": "~x86", "IUSE" : "flag", },
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/_ipc/
commit: 107dca9a3171e4870038769aef66612209ecd423 Author: Zac Medico gentoo org> AuthorDate: Tue May 1 17:27:06 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue May 1 17:27:06 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=107dca9a QueryCommand: handle empty *ROOT values for EAPI 7 The absence of a trailing slash in EAPI 7 means that it's valid for *ROOT values to be empty, so translate the empty value to a single slash where appropriate. This solves errors like the following: best_version: Invalid ROOT: ./ See: 6cb70f9ef0e7 ("Strip trailing slash from D, ED, ROOT, EROOT in EAPI 7") pym/portage/package/ebuild/_ipc/QueryCommand.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pym/portage/package/ebuild/_ipc/QueryCommand.py b/pym/portage/package/ebuild/_ipc/QueryCommand.py index 351c95628..fa6d1ea16 100644 --- a/pym/portage/package/ebuild/_ipc/QueryCommand.py +++ b/pym/portage/package/ebuild/_ipc/QueryCommand.py @@ -1,4 +1,4 @@ -# Copyright 2010-2013 Gentoo Foundation +# Copyright 2010-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -49,7 +49,7 @@ class QueryCommand(IpcCommand): db = self.get_db() eapi = self.settings.get('EAPI') - root = normalize_path(root).rstrip(os.path.sep) + os.path.sep + root = normalize_path(root or os.sep).rstrip(os.sep) + os.sep if root not in db: return ('', '%s: Invalid ROOT: %s\n' % (cmd, root), 3)
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/_config/, repoman/man/, pym/portage/package/ebuild/, ...
commit: d8d47bf451bb3fd6bce1cdb035a5f12253f5a167 Author: James Le Cuirot gentoo org> AuthorDate: Sat Sep 30 20:45:54 2017 + Commit: Zac Medico gentoo org> CommitDate: Tue May 1 08:28:43 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8d47bf4 Export BROOT to ebuild env in EAPI 7 Export the BROOT variable corresponding to the location where BDEPEND are installed. Bug: https://bugs.gentoo.org/317337 bin/eapi.sh| 4 bin/ebuild.sh | 3 +++ bin/phase-functions.sh | 3 +++ man/ebuild.5 | 6 ++ pym/portage/eapi.py| 7 ++- pym/portage/package/ebuild/_config/LocationsManager.py | 4 pym/portage/package/ebuild/_config/special_env_vars.py | 4 ++-- pym/portage/package/ebuild/config.py | 10 +- repoman/cnf/qa_data/qa_data.yaml | 2 +- repoman/man/repoman.1 | 2 +- .../pym/repoman/modules/linechecks/assignment/assignment.py| 9 - repoman/pym/repoman/modules/linechecks/quotes/quotes.py| 3 ++- 12 files changed, 49 insertions(+), 8 deletions(-) diff --git a/bin/eapi.sh b/bin/eapi.sh index b4d8c9a90..3f4c9691b 100644 --- a/bin/eapi.sh +++ b/bin/eapi.sh @@ -30,6 +30,10 @@ ___eapi_has_prefix_variables() { [[ ! ${1-${EAPI-0}} =~ ^(0|1|2)$ || " ${FEATURES} " == *" force-prefix "* ]] } +___eapi_has_BROOT() { + [[ ! ${1-${EAPI-0}} =~ ^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend|5-progress|6)$ ]] +} + ___eapi_has_SYSROOT() { [[ ! ${1-${EAPI-0}} =~ ^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend|5-progress|6)$ ]] } diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 11441bfb2..98ed570c2 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -769,6 +769,9 @@ else if ___eapi_has_prefix_variables; then declare -r ED EPREFIX EROOT fi + if ___eapi_has_BROOT; then + declare -r BROOT + fi # If ${EBUILD_FORCE_TEST} == 1 and USE came from ${T}/environment # then it might not have USE=test like it's supposed to here. diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh index 1e096cddc..1f9faaa41 100644 --- a/bin/phase-functions.sh +++ b/bin/phase-functions.sh @@ -100,6 +100,9 @@ __filter_readonly_variables() { filtered_vars="$readonly_bash_vars $bash_misc_vars $PORTAGE_READONLY_VARS $misc_garbage_vars" + if ___eapi_has_BROOT; then + filtered_vars+=" BROOT" + fi if ___eapi_has_SYSROOT; then filtered_vars+=" SYSROOT" fi diff --git a/man/ebuild.5 b/man/ebuild.5 index eb27d43bb..9f491dd73 100644 --- a/man/ebuild.5 +++ b/man/ebuild.5 @@ -527,6 +527,12 @@ Beginning with \fBEAPI 3\fR, contains "${ROOT%/}${EPREFIX}/" for convenience purposes. Do not modify this variable. .TP +.B BROOT\fR = \fI"${EPREFIX}" +Beginning with \fBEAPI 7\fR, the absolute path to the root directory +containing build dependencies satisfied by BDEPEND, typically +executable build tools. This includes any applicable offset prefix. Do +not modify this variable. +.TP .B DESCRIPTION\fR = \fI"A happy little package" Should contain a short description of the package. .TP diff --git a/pym/portage/eapi.py b/pym/portage/eapi.py index f34e19ac9..158d58243 100644 --- a/pym/portage/eapi.py +++ b/pym/portage/eapi.py @@ -123,13 +123,17 @@ def eapi_path_variables_end_with_trailing_slash(eapi): return eapi in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi", "5", "5-progress", "6") +def eapi_has_broot(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi", + "5", "5-progress", "5-hdepend", "6") + def eapi_has_sysroot(eapi): return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi", "5", "5-progress", "5-hdepend", "6") _eapi_attrs = collections.namedtuple('_eapi_attrs', 'allows_package_provided ' - 'bdepend dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC ' + 'bdepend broot dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC ' 'exports_PORTDIR exports_ECLASSDIR ' 'feature_flag_test feature_flag_targetroot ' 'hdepend iuse_defaults iuse_effective posixish_locale ' @@ -159,6 +163,7 @@ def _get_eapi_attrs(eapi): eapi_attrs = _eapi_attrs( allows_package_provided=(eapi is None or eapi_allows_package_provided(eapi)), bdepend = (eapi is not None and eapi_has_bdepend(eapi)), + broot = (eapi is None or eapi_has_broot(eapi)), dots_in_PN = (eapi is None or eapi_allows_dots_in_PN(eapi)), d
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/_config/, pym/_emerge/, pym/portage/, bin/, ...
commit: a41dacf7926c0242678300a8e1770df84029e8a4 Author: James Le Cuirot gentoo org> AuthorDate: Mon Apr 23 21:49:43 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue May 1 08:28:43 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a41dacf7 Export SYSROOT and ESYSROOT in ebuild env in EAPI 7 SYSROOT is the new destination for DEPEND, however it must currently be equal either / or ROOT. This is partly to simplify the implementation and partly because supporting a third separate destination would not be of much practical benefit anyway. Bug: https://bugs.gentoo.org/317337 bin/eapi.sh| 4 bin/phase-functions.sh | 6 ++ pym/_emerge/actions.py | 2 +- pym/_emerge/depgraph.py| 4 +++- pym/portage/__init__.py| 6 +++--- pym/portage/_legacy_globals.py | 3 ++- pym/portage/eapi.py| 7 ++- .../package/ebuild/_config/LocationsManager.py | 19 +++-- pym/portage/package/ebuild/config.py | 24 +++--- 9 files changed, 63 insertions(+), 12 deletions(-) diff --git a/bin/eapi.sh b/bin/eapi.sh index 40af7b776..b4d8c9a90 100644 --- a/bin/eapi.sh +++ b/bin/eapi.sh @@ -30,6 +30,10 @@ ___eapi_has_prefix_variables() { [[ ! ${1-${EAPI-0}} =~ ^(0|1|2)$ || " ${FEATURES} " == *" force-prefix "* ]] } +___eapi_has_SYSROOT() { + [[ ! ${1-${EAPI-0}} =~ ^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend|5-progress|6)$ ]] +} + ___eapi_has_HDEPEND() { [[ ${1-${EAPI-0}} =~ ^(5-hdepend)$ ]] } diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh index 3de8d01b5..1e096cddc 100644 --- a/bin/phase-functions.sh +++ b/bin/phase-functions.sh @@ -100,10 +100,16 @@ __filter_readonly_variables() { filtered_vars="$readonly_bash_vars $bash_misc_vars $PORTAGE_READONLY_VARS $misc_garbage_vars" + if ___eapi_has_SYSROOT; then + filtered_vars+=" SYSROOT" + fi # Don't filter/interfere with prefix variables unless they are # supported by the current EAPI. if ___eapi_has_prefix_variables; then filtered_vars+=" ED EPREFIX EROOT" + if ___eapi_has_SYSROOT; then + filtered_vars+=" ESYSROOT" + fi fi if ___eapi_has_PORTDIR_ECLASSDIR; then filtered_vars+=" PORTDIR ECLASSDIR" diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 1a289f523..70fb8d3b4 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -2438,7 +2438,7 @@ def load_emerge_config(emerge_config=None, env=None, **kargs): env = os.environ if env is None else env kwargs = {'env': env} for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT"), - ("eprefix", "EPREFIX")): + ("sysroot", "SYSROOT"), ("eprefix", "EPREFIX")): v = env.get(envvar) if v and v.strip(): kwargs[k] = v diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 67f912f5e..fbd16ad98 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -3322,8 +3322,10 @@ class depgraph(object): if removal_action: depend_root = myroot else: - if eapi_attrs.bdepend or eapi_attrs.hdepend: + if eapi_attrs.hdepend: depend_root = myroot + elif eapi_attrs.bdepend: + depend_root = pkg.root_config.settings["ESYSROOT"] else: depend_root = self._frozen_config._running_root.root root_deps = self._frozen_config.myopts.get("--root-deps") diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 137351a69..15c404f6f 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -525,7 +525,7 @@ class _trees_dict(dict): self._target_eroot = None def create_trees(config_root=None, target_root=None, trees=None, env=None, - eprefix=None): + sysroot=None, eprefix=None): if trees is None: trees = _trees_dict() @@ -538,7 +538,7 @@ def create_trees(config_root=None, target_root=None, trees=None, env=None, env = os.environ settings = config(config_root=config_root, target_root=target_root, - env=env, eprefix=eprefix) + env=env, sysroot=sysroot, eprefix=eprefix) settings.lock() depcachedir = settings.get('PORTAGE_DEPCACHEDIR') @@ -562,7 +562,7 @@ def create_trees(config_root=None, target_root=None, trees=None, env=None, if depc
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/, pym/portage/dbapi/, pym/portage/util/futures/, ...
commit: 90d78484d6be481a9caf22c017c62ea43f8ffe33 Author: Zac Medico gentoo org> AuthorDate: Tue May 1 06:22:10 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue May 1 06:46:27 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=90d78484 _PortageEventLoop: add _asyncio_* properties for internal use It's better to avoid accessing the _PortageEventLoop._loop attribute which exposes *all* EventLoop methods, therefore expose _asyncio_child_watcher and _asyncio_wrapper attributes for use by portage internals, providing minimal compatibility between _PortageEventloop and EventLoop. pym/portage/dbapi/porttree.py | 2 +- .../util/futures/asyncio/test_subprocess_exec.py | 2 +- .../tests/util/futures/test_iter_completed.py | 2 +- pym/portage/util/futures/executor/fork.py | 2 +- pym/portage/util/futures/iter_completed.py | 2 +- pym/portage/util/futures/unix_events.py| 22 ++ 6 files changed, 27 insertions(+), 5 deletions(-) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 3ce214cd7..6c38232bb 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -667,7 +667,7 @@ class portdbapi(dbapi): proc = EbuildMetadataPhase(cpv=mycpv, ebuild_hash=ebuild_hash, portdb=self, - repo_path=mylocation, scheduler=loop._loop, + repo_path=mylocation, scheduler=loop, settings=self.doebuild_settings) proc.addExitListener(functools.partial(self._aux_get_return, diff --git a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py index 8c8c395ca..be103a9e0 100644 --- a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py +++ b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py @@ -34,7 +34,7 @@ class _Reader(object): def __init__(self, future, input_file, loop): self._future = future self._pipe_reader = PipeReader( - input_files={'input_file':input_file}, scheduler=loop._loop) + input_files={'input_file':input_file}, scheduler=loop) self._future.add_done_callback(self._cancel_callback) self._pipe_reader.addExitListener(self._eof) diff --git a/pym/portage/tests/util/futures/test_iter_completed.py b/pym/portage/tests/util/futures/test_iter_completed.py index 71343c22d..90668eb02 100644 --- a/pym/portage/tests/util/futures/test_iter_completed.py +++ b/pym/portage/tests/util/futures/test_iter_completed.py @@ -46,7 +46,7 @@ class IterCompletedTestCase(TestCase): def future_generator(): for task in tasks: task.future = loop.create_future() - task.scheduler = loop._loop + task.scheduler = loop task.start() yield task.future diff --git a/pym/portage/util/futures/executor/fork.py b/pym/portage/util/futures/executor/fork.py index 51367f934..81c292e2c 100644 --- a/pym/portage/util/futures/executor/fork.py +++ b/pym/portage/util/futures/executor/fork.py @@ -54,7 +54,7 @@ class ForkExecutor(object): future, proc = self._submit_queue.popleft() future.add_done_callback(functools.partial(self._cancel_cb, proc)) proc.addExitListener(functools.partial(self._proc_exit, future)) - proc.scheduler = self._loop._loop + proc.scheduler = self._loop proc.start() self._running_tasks[id(proc)] = proc diff --git a/pym/portage/util/futures/iter_completed.py b/pym/portage/util/futures/iter_completed.py index 1d6a9a4bd..8b0f417d9 100644 --- a/pym/portage/util/futures/iter_completed.py +++ b/pym/portage/util/futures/iter_completed.py @@ -77,7 +77,7 @@ def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None): task_generator(), max_jobs=max_jobs, max_load=max_load, - event_loop=loop._loop) + event_loop=loop) def done_callback(future_done_set, wait_result): """Propagate results from wait_result to future_done_set.""" diff --git a/pym/portage/util/futures/unix_events.py b/pym/portage/util/futures/unix_events.py index 1a86ed439..00f522b61 100644 --- a/pym/portage/util/futures/unix_events.py +++ b/pym/portage/util/futures/unix_events.py @@ -72,6 +72,28 @@ class _PortageEventLoop(events.AbstractEventLoop): self.set_debug = loop.set_debug self.get_debug = loop.get_debug + @property + def _asyncio_child_watcher(self): +
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/, pym/portage/util/_async/
commit: 8b3dd3537a2c50035d6369946e65305d448d7f1b Author: Zac Medico gentoo org> AuthorDate: Mon Apr 30 19:11:52 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 30 19:25:20 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8b3dd353 AbstractPollTask: remove deprecated _read_* event parameter Fixes: c11a6ec05f02 ("AbstractPollTask: add_reader asyncio compat (bug 654382)") pym/_emerge/AbstractPollTask.py | 6 ++ pym/_emerge/AsynchronousLock.py | 2 +- pym/_emerge/EbuildIpcDaemon.py| 2 +- pym/_emerge/EbuildMetadataPhase.py| 2 +- pym/_emerge/PipeReader.py | 4 ++-- pym/portage/dbapi/_MergeProcess.py| 2 +- pym/portage/util/_async/PipeLogger.py | 2 +- 7 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py index f68cd3a15..6cccb27d9 100644 --- a/pym/_emerge/AbstractPollTask.py +++ b/pym/_emerge/AbstractPollTask.py @@ -19,7 +19,7 @@ class AbstractPollTask(AsynchronousTask): def isAlive(self): return bool(self._registered) - def _read_array(self, f, event): + def _read_array(self, f): """ NOTE: array.fromfile() is used here only for testing purposes, because it has bugs in all known versions of Python (including @@ -65,7 +65,7 @@ class AbstractPollTask(AsynchronousTask): return buf - def _read_buf(self, fd, event): + def _read_buf(self, fd): """ Read self._bufsize into a string of bytes, handling EAGAIN and EIO. This will only call os.read() once, so the caller should @@ -78,8 +78,6 @@ class AbstractPollTask(AsynchronousTask): @param fd: file descriptor (non-blocking mode required) @type fd: int - @param event: poll event flags - @type event: int @rtype: bytes or None @return: A string of bytes, or None """ diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py index 6cf37369f..aed1bcb15 100644 --- a/pym/_emerge/AsynchronousLock.py +++ b/pym/_emerge/AsynchronousLock.py @@ -266,7 +266,7 @@ class _LockProcess(AbstractPollTask): return self.returncode def _output_handler(self): - buf = self._read_buf(self._files['pipe_in'], None) + buf = self._read_buf(self._files['pipe_in']) if buf: self._acquired = True self._unregister() diff --git a/pym/_emerge/EbuildIpcDaemon.py b/pym/_emerge/EbuildIpcDaemon.py index c16049ee4..d0dbe18bd 100644 --- a/pym/_emerge/EbuildIpcDaemon.py +++ b/pym/_emerge/EbuildIpcDaemon.py @@ -34,7 +34,7 @@ class EbuildIpcDaemon(FifoIpcDaemon): def _input_handler(self): # Read the whole pickle in a single atomic read() call. - data = self._read_buf(self._files.pipe_in, None) + data = self._read_buf(self._files.pipe_in) if data is None: pass # EAGAIN elif data: diff --git a/pym/_emerge/EbuildMetadataPhase.py b/pym/_emerge/EbuildMetadataPhase.py index 42bcd6739..4940d40b6 100644 --- a/pym/_emerge/EbuildMetadataPhase.py +++ b/pym/_emerge/EbuildMetadataPhase.py @@ -130,7 +130,7 @@ class EbuildMetadataPhase(SubProcess): def _output_handler(self): while True: - buf = self._read_buf(self._files.ebuild, None) + buf = self._read_buf(self._files.ebuild) if buf is None: break # EAGAIN elif buf: diff --git a/pym/_emerge/PipeReader.py b/pym/_emerge/PipeReader.py index 6a3fc9ea6..151be94ea 100644 --- a/pym/_emerge/PipeReader.py +++ b/pym/_emerge/PipeReader.py @@ -61,7 +61,7 @@ class PipeReader(AbstractPollTask): def _output_handler(self, fd): while True: - data = self._read_buf(fd, None) + data = self._read_buf(fd) if data is None: break if data: @@ -79,7 +79,7 @@ class PipeReader(AbstractPollTask): break while True: - data = self._read_array(f, None) + data = self._read_array(f) if data is None: break if data: diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py index 42f2d84e5..fefdf8635 100644 --- a/pym/portage/dbapi/_MergeProcess.py +++ b/pym/portage/dbapi/_MergeProcess.py @@ -79,7 +79,7 @@ class MergeProcess(ForkProcess): self._locked_vdb = False def _elog_output_handler(self):
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
commit: 41af82685d688cb03da743cdd03295271a3ef09c Author: Zac Medico gentoo org> AuthorDate: Mon Apr 30 05:45:13 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 30 06:20:01 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=41af8268 _MergeProcess: add_reader asyncio compat (bug 654382) Use add_reader for asyncio compatibility. Bug: https://bugs.gentoo.org/654382 pym/portage/dbapi/_MergeProcess.py | 27 +++ 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py index bfbe387e4..42f2d84e5 100644 --- a/pym/portage/dbapi/_MergeProcess.py +++ b/pym/portage/dbapi/_MergeProcess.py @@ -7,7 +7,6 @@ import signal import sys import traceback -import errno import fcntl import portage from portage import os, _unicode_decode @@ -24,7 +23,7 @@ class MergeProcess(ForkProcess): __slots__ = ('mycat', 'mypkg', 'settings', 'treetype', 'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild', 'mydbapi', 'postinst_failure', 'prev_mtimes', 'unmerge', - '_elog_reader_fd', '_elog_reg_id', + '_elog_reader_fd', '_buf', '_elog_keys', '_locked_vdb') def _start(self): @@ -79,14 +78,8 @@ class MergeProcess(ForkProcess): self.vartree.dbapi.unlock() self._locked_vdb = False - def _elog_output_handler(self, fd, event): - output = None - if event & self.scheduler.IO_IN: - try: - output = os.read(fd, self._bufsize) - except OSError as e: - if e.errno not in (errno.EAGAIN, errno.EINTR): - raise + def _elog_output_handler(self): + output = self._read_buf(self._elog_reader_fd, None) if output: lines = _unicode_decode(output).split('\n') if len(lines) == 1: @@ -101,15 +94,12 @@ class MergeProcess(ForkProcess): reporter = getattr(portage.elog.messages, funcname) reporter(msg, phase=phase, key=key, out=out) - if event & self.scheduler.IO_HUP: - self.scheduler.source_remove(self._elog_reg_id) - self._elog_reg_id = None + elif output is not None: # EIO/POLLHUP + self.scheduler.remove_reader(self._elog_reader_fd) os.close(self._elog_reader_fd) self._elog_reader_fd = None return False - return True - def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call @@ -142,8 +132,7 @@ class MergeProcess(ForkProcess): treetype=self.treetype, vartree=self.vartree, blockers=blockers, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd - self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd, - self._registered_events, self._elog_output_handler) + self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an @@ -275,10 +264,8 @@ class MergeProcess(ForkProcess): pass self._unlock_vdb() - if self._elog_reg_id is not None: - self.scheduler.source_remove(self._elog_reg_id) - self._elog_reg_id = None if self._elog_reader_fd is not None: + self.scheduler.remove_reader(self._elog_reader_fd) os.close(self._elog_reader_fd) self._elog_reader_fd = None if self._elog_keys is not None:
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 2495fe6ff060e2ed8ee54e08a4dec132de1f4984 Author: Zac Medico gentoo org> AuthorDate: Mon Apr 30 05:52:07 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 30 06:20:01 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2495fe6f PipeLogger: add_reader asyncio compat (bug 654382) Use add_reader for asyncio compatibility. Bug: https://bugs.gentoo.org/654382 pym/portage/util/_async/PipeLogger.py | 20 ++-- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/pym/portage/util/_async/PipeLogger.py b/pym/portage/util/_async/PipeLogger.py index 02de74b16..61000271d 100644 --- a/pym/portage/util/_async/PipeLogger.py +++ b/pym/portage/util/_async/PipeLogger.py @@ -21,7 +21,7 @@ class PipeLogger(AbstractPollTask): """ __slots__ = ("input_fd", "log_file_path", "stdout_fd") + \ - ("_log_file", "_log_file_real", "_reg_id") + ("_log_file", "_log_file_real") def _start(self): @@ -57,8 +57,7 @@ class PipeLogger(AbstractPollTask): fcntl.fcntl(fd, fcntl.F_SETFD, fcntl.fcntl(fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - self._reg_id = self.scheduler.io_add_watch(fd, - self._registered_events, self._output_handler) + self.scheduler.add_reader(fd, self._output_handler, fd) self._registered = True def _cancel(self): @@ -66,14 +65,14 @@ class PipeLogger(AbstractPollTask): if self.returncode is None: self.returncode = self._cancelled_returncode - def _output_handler(self, fd, event): + def _output_handler(self, fd): background = self.background stdout_fd = self.stdout_fd log_file = self._log_file while True: - buf = self._read_buf(fd, event) + buf = self._read_buf(fd, None) if buf is None: # not a POLLIN event, EAGAIN, etc... @@ -124,20 +123,13 @@ class PipeLogger(AbstractPollTask): log_file.write(buf) log_file.flush() - self._unregister_if_appropriate(event) - - return True - def _unregister(self): - - if self._reg_id is not None: - self.scheduler.source_remove(self._reg_id) - self._reg_id = None - if self.input_fd is not None: if isinstance(self.input_fd, int): + self.scheduler.remove_reader(self.input_fd) os.close(self.input_fd) else: + self.scheduler.remove_reader(self.input_fd.fileno()) self.input_fd.close() self.input_fd = None
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/, pym/portage/util/_eventloop/
commit: c77afbc31fa687cc612a6f946b324bf4d74d8175 Author: Zac Medico gentoo org> AuthorDate: Mon Apr 30 01:49:18 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 30 02:14:41 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c77afbc3 EventLoop: call add_reader/writer callbacks after pipe is closed (bug 654382) Callbacks registered via add_reader/writer methods need to be called when the other end of a pipe is closed, which does not result in a normal read or write event. Therefore, respond to other event types as well, for compatibility with the asyncio event loop implementation. The included unit tests demonstrate asyncio compatible behavior for both reader and writer callbacks. Bug: https://bugs.gentoo.org/654382 .../tests/util/futures/asyncio/test_pipe_closed.py | 133 + pym/portage/util/_eventloop/EventLoop.py | 7 +- 2 files changed, 138 insertions(+), 2 deletions(-) diff --git a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py new file mode 100644 index 0..1ecddab78 --- /dev/null +++ b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py @@ -0,0 +1,133 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import os +import pty +import shutil +import socket +import sys +import tempfile + +from portage.tests import TestCase +from portage.util.futures import asyncio +from portage.util.futures.unix_events import ( + DefaultEventLoopPolicy, + _set_nonblocking, +) + + +class _PipeClosedTestCase(object): + + def test_pipe(self): + read_end, write_end = os.pipe() + self._do_test(read_end, write_end) + + def test_pty_device(self): + try: + read_end, write_end = pty.openpty() + except EnvironmentError: + self.skipTest('pty not available') + self._do_test(read_end, write_end) + + def test_domain_socket(self): + if sys.version_info >= (3, 2): + read_end, write_end = socket.socketpair() + else: + self.skipTest('socket detach not supported') + self._do_test(read_end.detach(), write_end.detach()) + + def test_named_pipe(self): + tempdir = tempfile.mkdtemp() + try: + fifo_path = os.path.join(tempdir, 'fifo') + os.mkfifo(fifo_path) + self._do_test(os.open(fifo_path, os.O_NONBLOCK|os.O_RDONLY), + os.open(fifo_path, os.O_NONBLOCK|os.O_WRONLY)) + finally: + shutil.rmtree(tempdir) + + +class ReaderPipeClosedTestCase(_PipeClosedTestCase, TestCase): + """ + Test that a reader callback is called after the other end of + the pipe has been closed. + """ + def _do_test(self, read_end, write_end): + initial_policy = asyncio.get_event_loop_policy() + if not isinstance(initial_policy, DefaultEventLoopPolicy): + asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) + + loop = asyncio.get_event_loop() + read_end = os.fdopen(read_end, 'rb', 0) + write_end = os.fdopen(write_end, 'wb', 0) + try: + def reader_callback(): + if not reader_callback.called.done(): + reader_callback.called.set_result(None) + + reader_callback.called = loop.create_future() + loop.add_reader(read_end.fileno(), reader_callback) + + # Allow the loop to check for IO events, and assert + # that our future is still not done. + loop.run_until_complete(asyncio.sleep(0, loop=loop)) + self.assertFalse(reader_callback.called.done()) + + # Demonstrate that the callback is called afer the + # other end of the pipe has been closed. + write_end.close() + loop.run_until_complete(reader_callback.called) + finally: + loop.remove_reader(read_end.fileno()) + write_end.close() + read_end.close() + asyncio.set_event_loop_policy(initial_policy) + + +class WriterPipeClosedTestCase(_PipeClosedTestCase, TestCase): + """ + Test that a writer callback is called after the other end of + the pipe has been closed. + """ + def _do_test(self, read_end, write_end): + initial_policy = asyncio.get_event_loop_policy() + if not isinstance(initial_policy, DefaultEven
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: abf4b7abcdc53bbbd86eb6f79d45417b54ed64d2 Author: Zac Medico gentoo org> AuthorDate: Sun Apr 29 22:12:13 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 29 22:18:28 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=abf4b7ab AsyncScheduler: call_later asyncio compat (bug 591760) Use call_later for asyncio compatibility. Also remove the timeout_add method from SchedulerInterface, since there are no more consumers. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/AsyncScheduler.py | 9 +++-- pym/portage/util/_async/SchedulerInterface.py | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py index 90e803eba..c6b523eaa 100644 --- a/pym/portage/util/_async/AsyncScheduler.py +++ b/pym/portage/util/_async/AsyncScheduler.py @@ -65,6 +65,11 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): task.addExitListener(self._task_exit) task.start() + if self._loadavg_check_id is not None: + self._loadavg_check_id.cancel() + self._loadavg_check_id = self._event_loop.call_later( + self._loadavg_latency, self._schedule) + # Triggers cleanup and exit listeners if there's nothing left to do. self.poll() @@ -80,14 +85,14 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): (self._max_jobs is True or self._max_jobs > 1): # We have to schedule periodically, in case the load # average has changed since the last call. - self._loadavg_check_id = self._event_loop.timeout_add( + self._loadavg_check_id = self._event_loop.call_later( self._loadavg_latency, self._schedule) self._schedule() def _cleanup(self): super(AsyncScheduler, self)._cleanup() if self._loadavg_check_id is not None: - self._event_loop.source_remove(self._loadavg_check_id) + self._loadavg_check_id.cancel() self._loadavg_check_id = None def _async_wait(self): diff --git a/pym/portage/util/_async/SchedulerInterface.py b/pym/portage/util/_async/SchedulerInterface.py index ff39bc587..56b844616 100644 --- a/pym/portage/util/_async/SchedulerInterface.py +++ b/pym/portage/util/_async/SchedulerInterface.py @@ -15,7 +15,6 @@ class SchedulerInterface(SlotObject): "IO_NVAL", "IO_OUT", "IO_PRI", "io_add_watch", "source_remove", - "timeout_add", "add_reader", "add_writer",
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
commit: 542c6e6c20a1d93a3a2af47c8de50eac3c891d5d Author: Zac Medico gentoo org> AuthorDate: Sun Apr 29 20:48:23 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 29 21:26:37 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=542c6e6c MergeProcess: fix deprecated _set_returncode (bug 654276) Move cleanup code from _set_returncode to _async_waitpid_cb, since _set_returncode expects an os.waitpid return value which is inconveniently different from the returncode that is passed to asyncio.AbstractChildWatcher.add_child_handler callbacks. Bug: https://bugs.gentoo.org/654276 pym/portage/dbapi/_MergeProcess.py | 10 +++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py index 35591af16..bfbe387e4 100644 --- a/pym/portage/dbapi/_MergeProcess.py +++ b/pym/portage/dbapi/_MergeProcess.py @@ -1,4 +1,4 @@ -# Copyright 2010-2013 Gentoo Foundation +# Copyright 2010-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io @@ -251,8 +251,12 @@ class MergeProcess(ForkProcess): # in order to avoid a race condition. os._exit(1) - def _set_returncode(self, wait_retval): - ForkProcess._set_returncode(self, wait_retval) + def _async_waitpid_cb(self, *args, **kwargs): + """ + Override _async_waitpid_cb to perform cleanup that is + not necessarily idempotent. + """ + ForkProcess._async_waitpid_cb(self, *args, **kwargs) if self.returncode == portage.const.RETURNCODE_POSTINST_FAILURE: self.postinst_failure = True self.returncode = os.EX_OK
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: be800bf0153a28ce034277d103a2021f93ac8b2e Author: Zac Medico gentoo org> AuthorDate: Sun Apr 29 05:10:27 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 29 05:58:58 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=be800bf0 PopenProcess: add_child_handler asyncio compat (bug 591760) Migrate to asyncio's AbstractChildWatcher.add_child_handler interface. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/PopenProcess.py | 31 +++ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/pym/portage/util/_async/PopenProcess.py b/pym/portage/util/_async/PopenProcess.py index 4344b1c9d..3fb60d527 100644 --- a/pym/portage/util/_async/PopenProcess.py +++ b/pym/portage/util/_async/PopenProcess.py @@ -1,4 +1,4 @@ -# Copyright 2012-2017 Gentoo Foundation +# Copyright 2012-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.SubProcess import SubProcess @@ -13,8 +13,7 @@ class PopenProcess(SubProcess): self._registered = True if self.pipe_reader is None: - self._reg_id = self.scheduler.child_watch_add( - self.pid, self._child_watch_cb) + self._async_waitpid() else: try: self.pipe_reader.scheduler = self.scheduler @@ -24,17 +23,25 @@ class PopenProcess(SubProcess): self.pipe_reader.start() def _pipe_reader_exit(self, pipe_reader): - self._reg_id = self.scheduler.child_watch_add( - self.pid, self._child_watch_cb) + self._async_waitpid() - def _child_watch_cb(self, pid, condition, user_data=None): - self._reg_id = None - self._waitpid_cb(pid, condition) - self.wait() + def _async_waitpid(self): + if self.returncode is None: + self.scheduler._asyncio_child_watcher.\ + add_child_handler(self.pid, self._async_waitpid_cb) + else: + self._unregister() + self._async_wait() - def _set_returncode(self, wait_retval): - SubProcess._set_returncode(self, wait_retval) + def _async_waitpid_cb(self, pid, returncode): if self.proc.returncode is None: # Suppress warning messages like this: # ResourceWarning: subprocess 1234 is still running - self.proc.returncode = self.returncode + self.proc.returncode = returncode + self._unregister() + self.returncode = returncode + self._async_wait() + + def _poll(self): + # Simply rely on _async_waitpid_cb to set the returncode. + return self.returncode
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, pym/_emerge/, pym/portage/_emirrordist/
commit: f01f504fc8d4ea7442379482889b1cee4be0fcf4 Author: Zac Medico gentoo org> AuthorDate: Sun Apr 29 01:28:42 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 29 04:31:45 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f01f504f AsynchronousTask: remove unused _wait() methods (bug 653856) Remove _wait() and _waitpid_loop() methods which are unused since AsynchronousTask.wait() now uses run_until_complete(self.async_wait()) instead. Also remove the iteration method from SchedulerInterface, since there are no more consumers. Bug: https://bugs.gentoo.org/653856 pym/_emerge/AbstractEbuildProcess.py| 27 pym/_emerge/AbstractPollTask.py | 8 +--- pym/_emerge/AsynchronousLock.py | 9 pym/_emerge/AsynchronousTask.py | 6 --- pym/_emerge/CompositeTask.py| 48 + pym/_emerge/FifoIpcDaemon.py| 13 +- pym/_emerge/PipeReader.py | 7 pym/_emerge/SpawnProcess.py | 12 +- pym/_emerge/SubProcess.py | 56 + pym/portage/_emirrordist/MirrorDistTask.py | 7 pym/portage/util/_async/AsyncScheduler.py | 25 +-- pym/portage/util/_async/AsyncTaskFuture.py | 8 pym/portage/util/_async/PipeLogger.py | 7 pym/portage/util/_async/PipeReaderBlockingIO.py | 10 - pym/portage/util/_async/SchedulerInterface.py | 1 - 15 files changed, 6 insertions(+), 238 deletions(-) diff --git a/pym/_emerge/AbstractEbuildProcess.py b/pym/_emerge/AbstractEbuildProcess.py index b10aa4bfa..03c834912 100644 --- a/pym/_emerge/AbstractEbuildProcess.py +++ b/pym/_emerge/AbstractEbuildProcess.py @@ -386,33 +386,6 @@ class AbstractEbuildProcess(SpawnProcess): if not self.cancelled: self._unexpected_exit() - def _wait(self): - """ - Override _wait to unlock self._build_dir if necessary. Normally, it - should already be unlocked, so this functions only as a failsafe. - Execution of the failsafe code will automatically become a fatal - error at the same time as event loop recursion is disabled. - """ - # SpawnProcess._wait() requires the pid, so wait here for the - # pid to become available. - while self._start_future is not None: - self.scheduler.run_until_complete(self._start_future) - - SpawnProcess._wait(self) - - if self._build_dir is not None: - self._build_dir_unlock = self._build_dir.async_unlock() - # Unlock only once. - self._build_dir = None - - if not (self._build_dir_unlock is None or - self._build_dir_unlock.done()): - # This will automatically become a fatal error at the same - # time as event loop recursion is disabled. - self.scheduler.run_until_complete(self._build_dir_unlock) - - return self.returncode - def _async_wait(self): """ Override _async_wait to asynchronously unlock self._build_dir diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py index 0aac97be5..0ce3594b4 100644 --- a/pym/_emerge/AbstractPollTask.py +++ b/pym/_emerge/AbstractPollTask.py @@ -1,4 +1,4 @@ -# Copyright 1999-2015 Gentoo Foundation +# Copyright 1999-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import array @@ -136,12 +136,6 @@ class AbstractPollTask(AsynchronousTask): self.returncode = self.returncode or os.EX_OK self._async_wait() - def _wait(self): - if self.returncode is not None: - return self.returncode - self._wait_loop() - return self.returncode - def _wait_loop(self, timeout=None): loop = getattr(self.scheduler, '_asyncio_wrapper', self.scheduler) tasks = [self.async_wait()] diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py index 2019adaab..c5991bcff 100644 --- a/pym/_emerge/AsynchronousLock.py +++ b/pym/_emerge/AsynchronousLock.py @@ -82,15 +82,6 @@ class AsynchronousLock(AsynchronousTask): self._imp.poll() return self.returncode - def _wait(self): - """ - Deprecated. Use _async_wait() instead. - """ - if self.returncode is not None: - return self.returncode - self.returncode = self._imp.wait() -
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 2ab6eed08c9eafa29de6100b19c9c072b17b9167 Author: Zac Medico gentoo org> AuthorDate: Sun Apr 29 04:06:25 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 29 04:06:25 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2ab6eed0 PipeReaderBlockingIO._reader_thread: asyncio compat (bug 653856) Use call_soon_threadsafe for asyncio compatibility, and also remove idle_add from SchedulerInterface since there are no more consumers. Bug: https://bugs.gentoo.org/653856 pym/portage/util/_async/PipeReaderBlockingIO.py | 3 +-- pym/portage/util/_async/SchedulerInterface.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pym/portage/util/_async/PipeReaderBlockingIO.py b/pym/portage/util/_async/PipeReaderBlockingIO.py index b426f7695..0727d2c6a 100644 --- a/pym/portage/util/_async/PipeReaderBlockingIO.py +++ b/pym/portage/util/_async/PipeReaderBlockingIO.py @@ -55,7 +55,7 @@ class PipeReaderBlockingIO(AbstractPollTask): del self._threads[f] if not self._threads: # Thread-safe callback to EventLoop - self.scheduler.idle_add(self._eof) + self.scheduler.call_soon_threadsafe(self._eof) break f.close() @@ -64,7 +64,6 @@ class PipeReaderBlockingIO(AbstractPollTask): if self.returncode is None: self.returncode = os.EX_OK self._async_wait() - return False def _cancel(self): self._terminate.set() diff --git a/pym/portage/util/_async/SchedulerInterface.py b/pym/portage/util/_async/SchedulerInterface.py index ab2a70852..acacf7c31 100644 --- a/pym/portage/util/_async/SchedulerInterface.py +++ b/pym/portage/util/_async/SchedulerInterface.py @@ -1,4 +1,4 @@ -# Copyright 2012-2013 Gentoo Foundation +# Copyright 2012-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import gzip @@ -14,7 +14,6 @@ class SchedulerInterface(SlotObject): _event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN", "IO_NVAL", "IO_OUT", "IO_PRI", "child_watch_add", - "idle_add", "io_add_watch", "iteration", "source_remove",
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 532366feaf8e3cb04e0d19d90498cfe0b9361ab1 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 28 23:27:49 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 28 23:27:49 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=532366fe PipeReaderBlockingIO: fix deprecated _wait usage (bug 653856) Use _async_wait() instead of wait(). Bug: https://bugs.gentoo.org/653856 pym/portage/util/_async/PipeReaderBlockingIO.py | 9 ++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pym/portage/util/_async/PipeReaderBlockingIO.py b/pym/portage/util/_async/PipeReaderBlockingIO.py index b06adf6ed..b426f7695 100644 --- a/pym/portage/util/_async/PipeReaderBlockingIO.py +++ b/pym/portage/util/_async/PipeReaderBlockingIO.py @@ -1,4 +1,4 @@ -# Copyright 2012 Gentoo Foundation +# Copyright 2012-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 try: @@ -63,7 +63,7 @@ class PipeReaderBlockingIO(AbstractPollTask): self._registered = False if self.returncode is None: self.returncode = os.EX_OK - self.wait() + self._async_wait() return False def _cancel(self): @@ -71,9 +71,12 @@ class PipeReaderBlockingIO(AbstractPollTask): self._registered = False if self.returncode is None: self.returncode = self._cancelled_returncode - self.wait() + self._async_wait() def _wait(self): + """ + Deprecated. Use _async_wait() instead. + """ if self.returncode is not None: return self.returncode self._wait_loop()
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 98e5852117cbda19b4f4266eb71a053dced64a88 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 28 23:17:30 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 28 23:17:30 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=98e58521 AsyncTaskFuture: fix deprecated _wait usage (bug 653856) Fix _done_callback to use _async_wait() instead of wait(). Bug: https://bugs.gentoo.org/653856 pym/portage/util/_async/AsyncTaskFuture.py | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_async/AsyncTaskFuture.py b/pym/portage/util/_async/AsyncTaskFuture.py index ce881ba8d..cdc842b05 100644 --- a/pym/portage/util/_async/AsyncTaskFuture.py +++ b/pym/portage/util/_async/AsyncTaskFuture.py @@ -28,9 +28,12 @@ class AsyncTaskFuture(AsynchronousTask): self.returncode = os.EX_OK else: self.returncode = 1 - self.wait() + self._async_wait() def _wait(self): + """ + Deprecated. Use _async_wait() instead. + """ if self.returncode is None: self.scheduler.run_until_complete(self.future) return self.returncode
[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/
commit: 1b0a408fcd556b7d9912856b852a331260d22022 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 28 22:48:51 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 28 22:48:51 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1b0a408f MirrorDistTask: fix super in _async_wait Fixes: 27801746e9aa ("MirrorDistTask: fix deprecated _wait usage (bug 653856)") pym/portage/_emirrordist/MirrorDistTask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/portage/_emirrordist/MirrorDistTask.py b/pym/portage/_emirrordist/MirrorDistTask.py index a0912d673..b8cde0af4 100644 --- a/pym/portage/_emirrordist/MirrorDistTask.py +++ b/pym/portage/_emirrordist/MirrorDistTask.py @@ -253,4 +253,4 @@ class MirrorDistTask(CompositeTask): Override _async_wait to call self._cleanup(). """ self._cleanup() - super(CompositeTask, self)._async_wait() + super(MirrorDistTask, self)._async_wait()
[gentoo-commits] proj/portage:master commit in: pym/portage/
commit: 97887768e3f5e8bc1b520b119fde97fed16cfe7d Author: Zac Medico gentoo org> AuthorDate: Sat Apr 28 22:59:21 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 28 23:05:25 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=97887768 PortageKeyError: fix __unicode__ AttributeError under python2.7 File "pym/portage/exception.py", line 32, in __unicode__ if isinstance(self.value, unicode): AttributeError: 'PortageKeyError' object has no attribute 'value' pym/portage/exception.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pym/portage/exception.py b/pym/portage/exception.py index 263cdf087..aed8beeb9 100644 --- a/pym/portage/exception.py +++ b/pym/portage/exception.py @@ -44,6 +44,9 @@ class PortageException(Exception): class PortageKeyError(KeyError, PortageException): __doc__ = KeyError.__doc__ + def __init__(self, value): + KeyError.__init__(self, value) + PortageException.__init__(self, value) class CorruptionError(PortageException): """Corruption indication"""
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 3d81af6440cfd20cd007568779d6129083f1de24 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 28 22:45:56 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 28 22:45:56 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d81af64 AsyncScheduler: fix deprecated _wait usage (bug 653856) Override AsynchronousTask._async_wait() for cleanup, since AsynchronousTask._wait() is deprecated. Bug: https://bugs.gentoo.org/653856 pym/portage/util/_async/AsyncScheduler.py | 13 +++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py index 1b870c771..2ddbaa885 100644 --- a/pym/portage/util/_async/AsyncScheduler.py +++ b/pym/portage/util/_async/AsyncScheduler.py @@ -29,12 +29,11 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): def _poll(self): if not (self._is_work_scheduled() or self._keep_scheduling()): - self._cleanup() - if self._error_count > 0: self.returncode = 1 else: self.returncode = os.EX_OK + self._async_wait() return self.returncode def _cancel(self): @@ -91,7 +90,17 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): self._event_loop.source_remove(self._loadavg_check_id) self._loadavg_check_id = None + def _async_wait(self): + """ + Override _async_wait to call self._cleanup(). + """ + self._cleanup() + super(AsyncScheduler, self)._async_wait() + def _wait(self): + """ + Deprecated. Use _async_wait() instead. + """ # Loop while there are jobs to be scheduled. while self._keep_scheduling(): self._event_loop.iteration()
[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/
commit: 27801746e9aa7e9ffc02408b2b005aa106c0d8a7 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 28 21:51:01 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 28 21:55:46 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=27801746 MirrorDistTask: fix deprecated _wait usage (bug 653856) Override AsynchronousTask._async_wait() for cleanup, since AsynchronousTask._wait() is deprecated. Bug: https://bugs.gentoo.org/653856 pym/portage/_emirrordist/MirrorDistTask.py | 18 ++ 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pym/portage/_emirrordist/MirrorDistTask.py b/pym/portage/_emirrordist/MirrorDistTask.py index 48d0f7cf2..a0912d673 100644 --- a/pym/portage/_emirrordist/MirrorDistTask.py +++ b/pym/portage/_emirrordist/MirrorDistTask.py @@ -1,4 +1,4 @@ -# Copyright 2013-2014 Gentoo Foundation +# Copyright 2013-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -46,7 +46,7 @@ class MirrorDistTask(CompositeTask): self._assert_current(fetch) if self._was_cancelled(): - self.wait() + self._async_wait() return if self._config.options.delete: @@ -63,7 +63,7 @@ class MirrorDistTask(CompositeTask): self._assert_current(deletion) if self._was_cancelled(): - self.wait() + self._async_wait() return self._post_deletion() @@ -80,7 +80,7 @@ class MirrorDistTask(CompositeTask): self.returncode = os.EX_OK self._current_task = None - self.wait() + self._async_wait() def _update_recycle_db(self): @@ -242,5 +242,15 @@ class MirrorDistTask(CompositeTask): self._async_wait() def _wait(self): + """ + Deprecated. Use _async_wait() instead. + """ CompositeTask._wait(self) self._cleanup() + + def _async_wait(self): + """ + Override _async_wait to call self._cleanup(). + """ + self._cleanup() + super(CompositeTask, self)._async_wait()
[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/
commit: 7c652d1b967f9c4c6a7fbd9fc5e46a0e57438a16 Author: Zac Medico gentoo org> AuthorDate: Wed Apr 25 06:42:10 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 28 13:56:42 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c652d1b FetchIterator: fix event loop recursion (bug 654038) Since construction of FetchTask instances requires results from aux_get calls that would trigger event loop recursion when executed synchronously, add an _async_fetch_tasks function to construct FetchTask instances asynchronously and return a Future. Use an _EbuildFetchTasks class to wait for the FetchTask instances to become available, and then execute them. Bug: https://bugs.gentoo.org/654038 pym/portage/_emirrordist/FetchIterator.py | 324 -- 1 file changed, 218 insertions(+), 106 deletions(-) diff --git a/pym/portage/_emirrordist/FetchIterator.py b/pym/portage/_emirrordist/FetchIterator.py index 38419799d..366453c12 100644 --- a/pym/portage/_emirrordist/FetchIterator.py +++ b/pym/portage/_emirrordist/FetchIterator.py @@ -1,4 +1,4 @@ -# Copyright 2013 Gentoo Foundation +# Copyright 2013-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import threading @@ -7,14 +7,18 @@ from portage import os from portage.checksum import (_apply_hash_filter, _filter_unaccelarated_hashes, _hash_filter) from portage.dep import use_reduce -from portage.exception import PortageException +from portage.exception import PortageException, PortageKeyError +from portage.util._async.AsyncTaskFuture import AsyncTaskFuture +from portage.util._async.TaskScheduler import TaskScheduler +from portage.util.futures.iter_completed import iter_gather from .FetchTask import FetchTask +from _emerge.CompositeTask import CompositeTask + class FetchIterator(object): def __init__(self, config): self._config = config - self._log_failure = config.log_failure self._terminated = threading.Event() def terminate(self): @@ -41,9 +45,6 @@ class FetchIterator(object): portdb = self._config.portdb get_repo_for_location = portdb.repositories.get_repo_for_location - file_owners = self._config.file_owners - file_failures = self._config.file_failures - restrict_mirror_exemptions = self._config.restrict_mirror_exemptions hash_filter = _hash_filter( portdb.settings.get("PORTAGE_CHECKSUM_FILTER", "")) @@ -59,110 +60,221 @@ class FetchIterator(object): # Reset state so the Manifest is pulled once # for this cp / tree combination. - digests = None repo_config = get_repo_for_location(tree) + digests_future = portdb._event_loop.create_future() for cpv in portdb.cp_list(cp, mytree=tree): if self._terminated.is_set(): return - try: - restrict, = portdb.aux_get(cpv, ("RESTRICT",), - mytree=tree) - except (KeyError, PortageException) as e: - self._log_failure("%s\t\taux_get exception %s" % - (cpv, e)) - continue - - # Here we use matchnone=True to ignore conditional parts - # of RESTRICT since they don't apply unconditionally. - # Assume such conditionals only apply on the client side. - try: - restrict = frozenset(use_reduce(restrict, - flat=True, matchnone=True)) - except PortageException as e: - self._log_failure("%s\t\tuse_reduce exception %s" % - (cpv, e)) - continue - - if "fetch" in restrict: - continue - - try: - uri_map = portdb.getFetchMap(cpv) - except PortageException as e: - self._log_failure("%s\t\tgetFetchMap exception %s" % -
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/dbapi/, pym/portage/package/ebuild/_parallel_manifest/, ...
commit: 3e77f0199cb401acf974089fb6aa378fd45d0e90 Author: Zac Medico gentoo org> AuthorDate: Tue Apr 24 06:54:05 2018 + Commit: Zac Medico gentoo org> CommitDate: Fri Apr 27 22:56:02 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3e77f019 ManifestScheduler: async fetchlist_dict (bug 653946) In order to avoid event loop recursion, pass fetchlist_dict to ManifestTask as a Future. Bug: https://bugs.gentoo.org/653946 pym/portage/dbapi/porttree.py | 70 ++ .../ebuild/_parallel_manifest/ManifestScheduler.py | 25 .../ebuild/_parallel_manifest/ManifestTask.py | 24 +++- pym/portage/tests/dbapi/test_portdb_cache.py | 3 +- 4 files changed, 105 insertions(+), 17 deletions(-) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 975f03d5e..3ce214cd7 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -37,6 +37,7 @@ from portage import _unicode_encode from portage import OrderedDict from portage.util._eventloop.EventLoop import EventLoop from portage.util._eventloop.global_event_loop import global_event_loop +from portage.util.futures.iter_completed import iter_gather from _emerge.EbuildMetadataPhase import EbuildMetadataPhase import os as _os @@ -1393,6 +1394,75 @@ class FetchlistDict(Mapping): if sys.hexversion >= 0x300: keys = __iter__ + +def _async_manifest_fetchlist(portdb, repo_config, cp, cpv_list=None, + max_jobs=None, max_load=None, loop=None): + """ + Asynchronous form of FetchlistDict, with max_jobs and max_load + parameters in order to control async_aux_get concurrency. + + @param portdb: portdbapi instance + @type portdb: portdbapi + @param repo_config: repository configuration for a Manifest + @type repo_config: RepoConfig + @param cp: cp for a Manifest + @type cp: str + @param cpv_list: list of ebuild cpv values for a Manifest + @type cpv_list: list + @param max_jobs: max number of futures to process concurrently (default + is multiprocessing.cpu_count()) + @type max_jobs: int + @param max_load: max load allowed when scheduling a new future, + otherwise schedule no more than 1 future at a time (default + is multiprocessing.cpu_count()) + @type max_load: int or float + @param loop: event loop + @type loop: EventLoop + @return: a Future resulting in a Mapping compatible with FetchlistDict + @rtype: asyncio.Future (or compatible) + """ + loop = loop or global_event_loop() + loop = getattr(loop, '_asyncio_wrapper', loop) + result = loop.create_future() + cpv_list = (portdb.cp_list(cp, mytree=repo_config.location) + if cpv_list is None else cpv_list) + + def gather_done(gather_result): + # All exceptions must be consumed from gather_result before this + # function returns, in order to avoid triggering the event loop's + # exception handler. + e = None + if not gather_result.cancelled(): + for future in gather_result.result(): + if (future.done() and not future.cancelled() and + future.exception() is not None): + e = future.exception() + + if result.cancelled(): + return + elif e is None: + result.set_result(dict((k, list(v.result())) + for k, v in zip(cpv_list, gather_result.result( + else: + result.set_exception(e) + + gather_result = iter_gather( + # Use a generator expression for lazy evaluation, so that iter_gather + # controls the number of concurrent async_fetch_map calls. + (portdb.async_fetch_map(cpv, mytree=repo_config.location, loop=loop) + for cpv in cpv_list), + max_jobs=max_jobs, + max_load=max_load, + loop=loop, + ) + + gather_result.add_done_callback(gather_done) + result.add_done_callback(lambda result: + gather_result.cancel() if result.cancelled() else None) + + return result + + def _parse_uri_map(cpv, metadata, use=None): myuris = use_reduce(metadata.get('SRC_URI', ''), diff --git a/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py b/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py index 38ac4825e..fabea9bc1 100644 --- a/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py +++ b/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py @@ -1,10 +1,10 @@ -# Copyright 2012-2013 Gentoo Foundation +# Copyright 2012-201
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/
commit: be61882996099322bb3a1e82e71f475b4141ad40 Author: Zac Medico gentoo org> AuthorDate: Tue Apr 24 23:28:08 2018 + Commit: Zac Medico gentoo org> CommitDate: Fri Apr 27 21:33:00 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=be618829 Add iter_gather function (bug 653946) This is similar to asyncio.gather, but takes an iterator of futures as input, and includes support for max_jobs and max_load parameters. For bug 653946, this will be used to asynchronously gather the results of the portdbapi.async_fetch_map calls that are required to generate a Manifest, while using the max_jobs parameter to limit the number of concurrent async_aux_get calls. Bug: https://bugs.gentoo.org/653946 pym/portage/util/futures/iter_completed.py | 73 ++ 1 file changed, 73 insertions(+) diff --git a/pym/portage/util/futures/iter_completed.py b/pym/portage/util/futures/iter_completed.py index 5ad075305..1d6a9a4bd 100644 --- a/pym/portage/util/futures/iter_completed.py +++ b/pym/portage/util/futures/iter_completed.py @@ -112,3 +112,76 @@ def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None): # cleanup in case of interruption by SIGINT, etc scheduler.cancel() scheduler.wait() + + +def iter_gather(futures, max_jobs=None, max_load=None, loop=None): + """ + This is similar to asyncio.gather, but takes an iterator of + futures as input, and includes support for max_jobs and max_load + parameters. + + @param futures: iterator of asyncio.Future (or compatible) + @type futures: iterator + @param max_jobs: max number of futures to process concurrently (default + is multiprocessing.cpu_count()) + @type max_jobs: int + @param max_load: max load allowed when scheduling a new future, + otherwise schedule no more than 1 future at a time (default + is multiprocessing.cpu_count()) + @type max_load: int or float + @param loop: event loop + @type loop: EventLoop + @return: a Future resulting in a list of done input futures, in the + same order that they were yielded from the input iterator + @rtype: asyncio.Future (or compatible) + """ + loop = loop or global_event_loop() + loop = getattr(loop, '_asyncio_wrapper', loop) + result = loop.create_future() + futures_list = [] + + def future_generator(): + for future in futures: + futures_list.append(future) + yield future + + completed_iter = async_iter_completed( + future_generator(), + max_jobs=max_jobs, + max_load=max_load, + loop=loop, + ) + + def handle_result(future_done_set): + if result.cancelled(): + if not future_done_set.cancelled(): + # All exceptions must be consumed from future_done_set, in order + # to avoid triggering the event loop's exception handler. + list(future.exception() for future in future_done_set.result() + if not future.cancelled()) + return + + try: + handle_result.current_task = next(completed_iter) + except StopIteration: + result.set_result(futures_list) + else: + handle_result.current_task.add_done_callback(handle_result) + + try: + handle_result.current_task = next(completed_iter) + except StopIteration: + handle_result.current_task = None + result.set_result(futures_list) + else: + handle_result.current_task.add_done_callback(handle_result) + + def cancel_callback(result): + if (result.cancelled() and + handle_result.current_task is not None and + not handle_result.current_task.done()): + handle_result.current_task.cancel() + + result.add_done_callback(cancel_callback) + + return result
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: a65d93bdeb4d964603cd0ce9b0a75a571c9bdefa Author: Zac Medico gentoo org> AuthorDate: Thu Apr 26 09:19:22 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 26 09:31:09 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a65d93bd SchedulerInterface: expose AbstractEventLoop methods (bug 591760) Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/SchedulerInterface.py | 34 +++ 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/pym/portage/util/_async/SchedulerInterface.py b/pym/portage/util/_async/SchedulerInterface.py index 21420ae41..ab2a70852 100644 --- a/pym/portage/util/_async/SchedulerInterface.py +++ b/pym/portage/util/_async/SchedulerInterface.py @@ -13,10 +13,36 @@ class SchedulerInterface(SlotObject): _event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN", "IO_NVAL", "IO_OUT", "IO_PRI", - "call_soon", "call_soon_threadsafe", - "child_watch_add", "create_future", - "idle_add", "io_add_watch", "iteration", "run_until_complete", - "source_remove", "timeout_add") + "child_watch_add", + "idle_add", + "io_add_watch", + "iteration", + "source_remove", + "timeout_add", + + "add_reader", + "add_writer", + "call_at", + "call_exception_handler", + "call_later", + "call_soon", + "call_soon_threadsafe", + "close", + "create_future", + "default_exception_handler", + "get_debug", + "is_closed", + "is_running", + "remove_reader", + "remove_writer", + "run_in_executor", + "run_until_complete", + "set_debug", + "time", + + "_asyncio_child_watcher", + "_asyncio_wrapper", + ) __slots__ = _event_loop_attrs + ("_event_loop", "_is_background")
[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/
commit: 391daef6fce981acd5a01e41a0f7238044c48877 Author: Zac Medico gentoo org> AuthorDate: Thu Apr 26 08:29:04 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 26 08:43:53 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=391daef6 MirrorDistTask._term_callback(): use _async_wait() (bug 591760) Use _async_wait() to avoid event loop recursion, but don't call it prematurely, since since that could trigger event loop recursion if the current (cancelled) task's exit callback does not set the returncode first. Bug: https://bugs.gentoo.org/591760 pym/portage/_emirrordist/MirrorDistTask.py | 10 +- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pym/portage/_emirrordist/MirrorDistTask.py b/pym/portage/_emirrordist/MirrorDistTask.py index a34f2c061..48d0f7cf2 100644 --- a/pym/portage/_emirrordist/MirrorDistTask.py +++ b/pym/portage/_emirrordist/MirrorDistTask.py @@ -231,7 +231,15 @@ class MirrorDistTask(CompositeTask): if self._fetch_iterator is not None: self._fetch_iterator.terminate() self.cancel() - self.wait() + if self.returncode is None: + # In this case, the exit callback for self._current_task will + # trigger notification of exit listeners. Don't call _async_wait() + # yet, since that could trigger event loop recursion if the + # current (cancelled) task's exit callback does not set the + # returncode first. + pass + else: + self._async_wait() def _wait(self): CompositeTask._wait(self)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: d16206ccd315edbd4bd7a46c0b1aa9e59a46db43 Author: Zac Medico gentoo org> AuthorDate: Thu Apr 26 07:45:32 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 26 08:02:28 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d16206cc AsyncScheduler._poll(): fix event loop recursion (bug 591760) Call self._cleanup() and set the returncode, in order to avoid event loop recursion in self.wait(). Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/AsyncScheduler.py | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py index b89b57dab..1b870c771 100644 --- a/pym/portage/util/_async/AsyncScheduler.py +++ b/pym/portage/util/_async/AsyncScheduler.py @@ -29,7 +29,12 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): def _poll(self): if not (self._is_work_scheduled() or self._keep_scheduling()): - self.wait() + self._cleanup() + + if self._error_count > 0: + self.returncode = 1 + else: + self.returncode = os.EX_OK return self.returncode def _cancel(self):
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 384b61dd24a21aaccd7c643e58b07ea56c3cbdfc Author: Zac Medico gentoo org> AuthorDate: Thu Apr 26 05:31:54 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 26 05:46:30 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=384b61dd PipeLogger._output_handler: use _async_wait() (bug 591760) Use _async_wait() to avoid event loop recursion. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/PipeLogger.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/_async/PipeLogger.py b/pym/portage/util/_async/PipeLogger.py index aa605d94d..ed1202edf 100644 --- a/pym/portage/util/_async/PipeLogger.py +++ b/pym/portage/util/_async/PipeLogger.py @@ -1,4 +1,4 @@ -# Copyright 2008-2013 Gentoo Foundation +# Copyright 2008-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import fcntl @@ -89,7 +89,8 @@ class PipeLogger(AbstractPollTask): if not buf: # EOF self._unregister() - self.wait() + self.returncode = self.returncode or os.EX_OK + self._async_wait() break else:
[gentoo-commits] proj/portage:master commit in: pym/portage/util/, pym/portage/util/_async/, pym/_emerge/
commit: 6c865d6b3130de63dc3d534d2a0596dd59d5e662 Author: Zac Medico gentoo org> AuthorDate: Tue Apr 24 02:21:59 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 26 03:19:21 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c865d6b AsynchronousTask: add scheduler attribute (bug 653856) Add an AsynchronousTask.scheduler attribute, which will hold an event loop instance in order to work with Future instances. Name the attribute "scheduler" because this is the name used by many existing subclasses of AsynchronousTask. Since the AsyncScheduler class already has an _event_loop attribute that it inherits from PollScheduler, use a property for compatibility. Also update the SlotObject constructor to allow a subclass to override an attribute from __slots__ with a property, so that AsyncScheduler can use a property for the scheduler attribute. Bug: https://bugs.gentoo.org/653856 pym/_emerge/AbstractPollTask.py| 3 +-- pym/_emerge/AsynchronousLock.py| 2 +- pym/_emerge/AsynchronousTask.py| 2 +- pym/_emerge/CompositeTask.py | 2 +- pym/portage/util/SlotObject.py | 9 - pym/portage/util/_async/AsyncScheduler.py | 7 +++ pym/portage/util/_async/AsyncTaskFuture.py | 2 +- 7 files changed, 20 insertions(+), 7 deletions(-) diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py index 0d38bd481..aa8fc6593 100644 --- a/pym/_emerge/AbstractPollTask.py +++ b/pym/_emerge/AbstractPollTask.py @@ -11,8 +11,7 @@ from _emerge.AsynchronousTask import AsynchronousTask class AbstractPollTask(AsynchronousTask): - __slots__ = ("scheduler",) + \ - ("_registered",) + __slots__ = ("_registered",) _bufsize = 4096 diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py index 78ab37ecb..0178feab2 100644 --- a/pym/_emerge/AsynchronousLock.py +++ b/pym/_emerge/AsynchronousLock.py @@ -37,7 +37,7 @@ class AsynchronousLock(AsynchronousTask): signals to the main thread). """ - __slots__ = ('path', 'scheduler',) + \ + __slots__ = ('path',) + \ ('_imp', '_force_async', '_force_dummy', '_force_process', \ '_force_thread', '_unlock_future') diff --git a/pym/_emerge/AsynchronousTask.py b/pym/_emerge/AsynchronousTask.py index da58261db..e29324440 100644 --- a/pym/_emerge/AsynchronousTask.py +++ b/pym/_emerge/AsynchronousTask.py @@ -16,7 +16,7 @@ class AsynchronousTask(SlotObject): the task is complete and self.returncode has been set. """ - __slots__ = ("background", "cancelled", "returncode") + \ + __slots__ = ("background", "cancelled", "returncode", "scheduler") + \ ("_exit_listeners", "_exit_listener_stack", "_start_listeners", "_waiting") diff --git a/pym/_emerge/CompositeTask.py b/pym/_emerge/CompositeTask.py index f3acc9696..bfd4bacbd 100644 --- a/pym/_emerge/CompositeTask.py +++ b/pym/_emerge/CompositeTask.py @@ -6,7 +6,7 @@ from portage import os class CompositeTask(AsynchronousTask): - __slots__ = ("scheduler",) + ("_current_task",) + __slots__ = ("_current_task",) _TASK_QUEUED = -1 diff --git a/pym/portage/util/SlotObject.py b/pym/portage/util/SlotObject.py index 4bb682258..ba6215874 100644 --- a/pym/portage/util/SlotObject.py +++ b/pym/portage/util/SlotObject.py @@ -20,7 +20,14 @@ class SlotObject(object): raise AssertionError( "class '%s' duplicates '%s' value in __slots__ of base class '%s'" % (self.__class__.__name__, myattr, c.__name__)) - setattr(self, myattr, myvalue) + try: + setattr(self, myattr, myvalue) + except AttributeError: + # Allow a property to override a __slots__ value, but raise an + # error if the intended value is something other than None. + if not (myvalue is None and + isinstance(getattr(type(self), myattr, None), property)): + raise if kwargs: raise TypeError( diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py index 9beb8a848..b89b57dab 100644 --- a/pym/portage/util/_async/AsyncScheduler.py +++ b/pym/portage/util/_async/AsyncScheduler.py @@ -20,6 +20,13 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): self._remaining_tasks = True self._loadavg_check_id = None + @property + def scheduler(self): + """ +
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/ebuild/, pym/_emerge/
commit: 71a5a82313226f7be0d966d49392a53139a96f6b Author: Zac Medico gentoo org> AuthorDate: Tue Apr 24 02:47:11 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 26 03:19:22 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=71a5a823 AsynchronousTask: add async_wait() method (bug 653856) Since the AsynchronousTask.wait() method is prone to event loop recursion, deprecate it, and add an async_wait() method method to replace it. Instead of using task.wait() in order to implicitly run the event loop, now loop.run_until_complete(task.async_wait()) will be used to explicitly run the event loop. This explicit approach will make it more obvious when code will trigger event loop recursion which would not be compatible with asyncio's default event loop. Bug: https://bugs.gentoo.org/653856 pym/_emerge/AsynchronousTask.py | 23 +++ pym/portage/tests/ebuild/test_ipc_daemon.py | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pym/_emerge/AsynchronousTask.py b/pym/_emerge/AsynchronousTask.py index e29324440..7d2e6253b 100644 --- a/pym/_emerge/AsynchronousTask.py +++ b/pym/_emerge/AsynchronousTask.py @@ -29,6 +29,26 @@ class AsynchronousTask(SlotObject): self._start_hook() self._start() + def async_wait(self): + """ + Wait for returncode asynchronously. Notification is available + via the add_done_callback method of the returned Future instance. + + @returns: Future, result is self.returncode + """ + waiter = self.scheduler.create_future() + exit_listener = lambda self: waiter.set_result(self.returncode) + self.addExitListener(exit_listener) + waiter.add_done_callback(lambda waiter: + self.removeExitListener(exit_listener) if waiter.cancelled() else None) + if self.returncode is not None: + # If the returncode is not None, it means the exit event has already + # happened, so use _async_wait() to guarantee that the exit_listener + # is called. This does not do any harm because a given exit listener + # is never called more than once. + self._async_wait() + return waiter + def _start(self): self.returncode = os.EX_OK self.wait() @@ -47,6 +67,9 @@ class AsynchronousTask(SlotObject): return self.returncode def wait(self): + """ + Deprecated. Use async_wait() instead. + """ if self.returncode is None: if not self._waiting: self._waiting = True diff --git a/pym/portage/tests/ebuild/test_ipc_daemon.py b/pym/portage/tests/ebuild/test_ipc_daemon.py index bc18cdf64..e6da51a76 100644 --- a/pym/portage/tests/ebuild/test_ipc_daemon.py +++ b/pym/portage/tests/ebuild/test_ipc_daemon.py @@ -157,6 +157,6 @@ class IpcDaemonTestCase(TestCase): try: task_scheduler.start() event_loop.run_until_complete(self._run_done) - task_scheduler.wait() + event_loop.run_until_complete(task_scheduler.async_wait()) finally: timeout_handle.cancel()
[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/
commit: a7c8ddd9968741fd51ba21500a9f6b9ef27bef15 Author: Zac Medico gentoo org> AuthorDate: Wed Apr 25 07:28:44 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed Apr 25 07:28:44 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7c8ddd9 FetchTask._fetch_uri: use _async_wait() (bug 591760) Use _async_wait() to avoid event loop recursion in emirrordist --dry-run mode. Bug: https://bugs.gentoo.org/591760 pym/portage/_emirrordist/FetchTask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py index 47908cb6b..1440b697c 100644 --- a/pym/portage/_emirrordist/FetchTask.py +++ b/pym/portage/_emirrordist/FetchTask.py @@ -404,7 +404,7 @@ class FetchTask(CompositeTask): (self.distfile, uri)) self._success() self.returncode = os.EX_OK - self.wait() + self._async_wait() return if self.config.options.temp_dir:
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
commit: 698bbcc494e06719cdbb0522aff6a5fb0d249b36 Author: Zac Medico gentoo org> AuthorDate: Tue Apr 24 06:31:32 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue Apr 24 06:55:20 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=698bbcc4 portdbapi.async_fetch_map: fix future.exception() reference Fixes: 8002d9343c38 ("portdbapi: add async_fetch_map method (bug 653810)") pym/portage/dbapi/porttree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 8a5fc690c..975f03d5e 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -759,7 +759,7 @@ class portdbapi(dbapi): if result.cancelled(): return if aux_get_future.exception() is not None: - if isinstance(future.exception(), PortageKeyError): + if isinstance(aux_get_future.exception(), PortageKeyError): # Convert this to an InvalidDependString exception since # callers already handle it. result.set_exception(portage.exception.InvalidDependString(
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
commit: 508c7667bb0755c9f72b443879dcf7f631434fda Author: Zac Medico gentoo org> AuthorDate: Mon Apr 23 19:51:09 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 23 19:53:47 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=508c7667 portdbapi.async_fetch_map: pass loop to async_aux_get Fixes: 8002d9343c38 ("portdbapi: add async_fetch_map method (bug 653810)") pym/portage/dbapi/porttree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 3cd929963..8a5fc690c 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -783,7 +783,7 @@ class portdbapi(dbapi): {'EAPI':eapi,'SRC_URI':myuris}, use=useflags)) aux_get_future = self.async_aux_get( - mypkg, ["EAPI", "SRC_URI"], mytree=mytree) + mypkg, ["EAPI", "SRC_URI"], mytree=mytree, loop=loop) result.add_done_callback(lambda result: aux_get_future.cancel() if result.cancelled() else None) aux_get_future.add_done_callback(aux_get_done)
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
commit: 8002d9343c385075ece19a131b0f584ec255a5b5 Author: Zac Medico gentoo org> AuthorDate: Sun Apr 22 19:57:09 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 23 18:39:16 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8002d934 portdbapi: add async_fetch_map method (bug 653810) Add a portdbapi.async_fetch_map method which is identical to the existing portdbapi.getFetchMap method, but returns a future. This will be used by EbuildFetcher in order to avoid event loop recursion. Bug: https://bugs.gentoo.org/653810 Reviewed-by: Brian Dolbec gentoo.org> pym/portage/dbapi/porttree.py | 75 +-- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 951e5760a..3cd929963 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -728,25 +728,66 @@ class portdbapi(dbapi): URIs. @rtype: dict """ + loop = self._event_loop + return loop.run_until_complete( + self.async_fetch_map(mypkg, useflags=useflags, + mytree=mytree, loop=loop)) - try: - eapi, myuris = self.aux_get(mypkg, - ["EAPI", "SRC_URI"], mytree=mytree) - except KeyError: - # Convert this to an InvalidDependString exception since callers - # already handle it. - raise portage.exception.InvalidDependString( - "getFetchMap(): aux_get() error reading "+mypkg+"; aborting.") + def async_fetch_map(self, mypkg, useflags=None, mytree=None, loop=None): + """ + Asynchronous form of getFetchMap. - if not eapi_is_supported(eapi): - # Convert this to an InvalidDependString exception - # since callers already handle it. - raise portage.exception.InvalidDependString( - "getFetchMap(): '%s' has unsupported EAPI: '%s'" % \ - (mypkg, eapi)) - - return _parse_uri_map(mypkg, {'EAPI':eapi,'SRC_URI':myuris}, - use=useflags) + @param mypkg: cpv for an ebuild + @type mypkg: String + @param useflags: a collection of enabled USE flags, for evaluation of + conditionals + @type useflags: set, or None to enable all conditionals + @param mytree: The canonical path of the tree in which the ebuild + is located, or None for automatic lookup + @type mypkg: String + @param loop: event loop (defaults to global event loop) + @type loop: EventLoop + @return: A future that results in a dict which maps each file name to + a set of alternative URIs. + @rtype: asyncio.Future (or compatible) + """ + loop = loop or global_event_loop() + loop = getattr(loop, '_asyncio_wrapper', loop) + result = loop.create_future() + + def aux_get_done(aux_get_future): + if result.cancelled(): + return + if aux_get_future.exception() is not None: + if isinstance(future.exception(), PortageKeyError): + # Convert this to an InvalidDependString exception since + # callers already handle it. + result.set_exception(portage.exception.InvalidDependString( + "getFetchMap(): aux_get() error reading " + + mypkg + "; aborting.")) + else: + result.set_exception(future.exception()) + return + + eapi, myuris = aux_get_future.result() + + if not eapi_is_supported(eapi): + # Convert this to an InvalidDependString exception + # since callers already handle it. + result.set_exception(portage.exception.InvalidDependString( + "getFetchMap(): '%s' has unsupported EAPI: '%s'" % \ + (mypkg, eapi))) + return + + result.set_result(_parse_uri_map(mypkg, + {'EAPI':eapi,'SRC_URI':myuris}, use=useflags)) + + aux_get_future = self.async_aux_get( +
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/asyncio/
commit: c23d093d0330a9318534a1c521fb00d6360fabc0 Author: Zac Medico gentoo org> AuthorDate: Mon Apr 23 18:20:25 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 23 18:24:11 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c23d093d tests: prove run_until_complete executes done callbacks Prove that done callbacks are executed before run_until_complete returns, which is how asyncio's default event loop behaves. This behavior was implemented in portage's internal event loop in commit 25245d7eb86ed197b7d7cfead0dbe4ce8ad4bc5b. Bug: https://bugs.gentoo.org/591760 .../futures/asyncio/test_run_until_complete.py | 29 ++ 1 file changed, 29 insertions(+) diff --git a/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py b/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py new file mode 100644 index 0..fc8f198ca --- /dev/null +++ b/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py @@ -0,0 +1,29 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.util.futures import asyncio +from portage.util.futures.unix_events import DefaultEventLoopPolicy + + +class RunUntilCompleteTestCase(TestCase): + def test_add_done_callback(self): + initial_policy = asyncio.get_event_loop_policy() + if not isinstance(initial_policy, DefaultEventLoopPolicy): + asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) + + try: + loop = asyncio.get_event_loop() + f1 = loop.create_future() + f2 = loop.create_future() + f1.add_done_callback(f2.set_result) + loop.call_soon(lambda: f1.set_result(None)) + loop.run_until_complete(f1) + self.assertEqual(f1.done(), True) + + # This proves that done callbacks of f1 are executed before + # loop.run_until_complete(f1) returns, which is how asyncio's + # default event loop behaves. + self.assertEqual(f2.done(), True) + finally: + asyncio.set_event_loop_policy(initial_policy)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 3eb46b05eb098507ea74108559bf77006e0f8142 Author: Zac Medico gentoo org> AuthorDate: Mon Apr 23 00:49:15 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 23 00:49:15 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3eb46b05 FileDigester: use _async_waitpid (bug 591760) When pid exit status is not yet available, use the Subprocess _async_waitpid() method to avoid event loop recursion. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/FileDigester.py | 7 +-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/_async/FileDigester.py b/pym/portage/util/_async/FileDigester.py index 881c69280..72f06759c 100644 --- a/pym/portage/util/_async/FileDigester.py +++ b/pym/portage/util/_async/FileDigester.py @@ -60,8 +60,11 @@ class FileDigester(ForkProcess): def _digest_pipe_reader_exit(self, pipe_reader): self._parse_digests(pipe_reader.getvalue()) self._digest_pipe_reader = None - self._unregister() - self.wait() + if self.pid is None: + self._unregister() + self._async_wait() + else: + self._async_waitpid() def _unregister(self): ForkProcess._unregister(self)
[gentoo-commits] proj/portage:master commit in: pym/portage/package/ebuild/, pym/_emerge/
commit: db4dca876cdb004b12a944f5323a51bc4bbde770 Author: Zac Medico gentoo org> AuthorDate: Mon Apr 23 00:04:29 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 23 00:06:35 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=db4dca87 EbuildBuild._start(): fix event loop recursion (bug 653844) Use async_aux_get to fetch SRC_URI metadata (it's not cached in self.pkg.metadata because some packages have an extremely large SRC_URI value). Bug: https://bugs.gentoo.org/653844 pym/_emerge/EbuildBuild.py | 19 ++- pym/portage/package/ebuild/doebuild.py | 6 -- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index 9d4afd0ea..8ad8bcae9 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -35,23 +35,32 @@ class EbuildBuild(CompositeTask): ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): - - pkg = self.pkg - settings = self.settings - if not self.opts.fetchonly: - rval = _check_temp_dir(settings) + rval = _check_temp_dir(self.settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self._async_wait() return + # First get the SRC_URI metadata (it's not cached in self.pkg.metadata + # because some packages have an extremely large SRC_URI value). + self._start_task( + AsyncTaskFuture( + future=self.pkg.root_config.trees["porttree"].dbapi.\ + async_aux_get(self.pkg.cpv, ["SRC_URI"], myrepo=self.pkg.repo)), + self._start_with_metadata) + + def _start_with_metadata(self, aux_get_task): + self._assert_current(aux_get_task) + pkg = self.pkg + settings = self.settings root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) + settings.configdict["pkg"]["SRC_URI"], = aux_get_task.future.result() settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index 3c8414387..0dabafeb7 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -450,8 +450,10 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, if hasattr(mydbapi, "getFetchMap") and \ ("A" not in mysettings.configdict["pkg"] or \ "AA" not in mysettings.configdict["pkg"]): - src_uri, = mydbapi.aux_get(mysettings.mycpv, - ["SRC_URI"], mytree=mytree) + src_uri = mysettings.configdict["pkg"].get("SRC_URI") + if src_uri is None: + src_uri, = mydbapi.aux_get(mysettings.mycpv, + ["SRC_URI"], mytree=mytree) metadata = { "EAPI": eapi, "SRC_URI" : src_uri,
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: d836b0d32797f1431ebeaa20e2a18ea5cb7c3f7c Author: Zac Medico gentoo org> AuthorDate: Sun Apr 22 15:56:29 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 22 17:48:52 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d836b0d3 AsyncFunction: use _async_waitpid (bug 591760) When pid exit status is not yet available, use the Subprocess _async_waitpid() method to avoid event loop recursion. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/AsyncFunction.py | 7 +-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/_async/AsyncFunction.py b/pym/portage/util/_async/AsyncFunction.py index 40f6c5e75..ad3d8333f 100644 --- a/pym/portage/util/_async/AsyncFunction.py +++ b/pym/portage/util/_async/AsyncFunction.py @@ -57,8 +57,11 @@ class AsyncFunction(ForkProcess): # and returned an unsuccessful returncode. pass self._async_func_reader = None - self._unregister() - self.wait() + if self.returncode is None: + self._async_waitpid() + else: + self._unregister() + self._async_wait() def _unregister(self): ForkProcess._unregister(self)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, pym/portage/tests/util/futures/
commit: 6529d0e0ec017c4fa3048235649d25e71c967941 Author: Zac Medico gentoo org> AuthorDate: Sun Apr 22 17:27:32 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 22 17:27:32 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6529d0e0 test_iter_completed: fix max_load in testAsyncCancel Since async_iter_completed sets a default multiprocessing.cpu_count() value when max_load is None, use max_load=True to disable the load limit. Fixes: a9e8ebaa6979 ("Add async_iter_completed for asyncio migration (bug 591760)") pym/portage/tests/util/futures/test_iter_completed.py | 2 +- pym/portage/util/_async/AsyncScheduler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pym/portage/tests/util/futures/test_iter_completed.py b/pym/portage/tests/util/futures/test_iter_completed.py index b07146ed3..71343c22d 100644 --- a/pym/portage/tests/util/futures/test_iter_completed.py +++ b/pym/portage/tests/util/futures/test_iter_completed.py @@ -69,7 +69,7 @@ class IterCompletedTestCase(TestCase): yield future for future_done_set in async_iter_completed(future_generator(), - max_jobs=True, max_load=None, loop=loop): + max_jobs=True, max_load=True, loop=loop): future_done_set.cancel() break diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py index 3deb6cb04..9beb8a848 100644 --- a/pym/portage/util/_async/AsyncScheduler.py +++ b/pym/portage/util/_async/AsyncScheduler.py @@ -14,7 +14,7 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): if max_jobs is None: max_jobs = 1 self._max_jobs = max_jobs - self._max_load = max_load + self._max_load = None if max_load is True else max_load self._error_count = 0 self._running_tasks = set() self._remaining_tasks = True
[gentoo-commits] proj/portage:master commit in: pym/portage/tests/util/futures/
commit: a6e9c7cf429741015e26b923c8036416cc6bff7d Author: Zac Medico gentoo org> AuthorDate: Sun Apr 22 16:19:27 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 22 16:24:37 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a6e9c7cf test_iter_completed: fix SleepProcess._future_done cancel race Fixes: a9e8ebaa6979 ("Add async_iter_completed for asyncio migration (bug 591760)") pym/portage/tests/util/futures/test_iter_completed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pym/portage/tests/util/futures/test_iter_completed.py b/pym/portage/tests/util/futures/test_iter_completed.py index 1344523c6..b07146ed3 100644 --- a/pym/portage/tests/util/futures/test_iter_completed.py +++ b/pym/portage/tests/util/futures/test_iter_completed.py @@ -19,7 +19,8 @@ class SleepProcess(ForkProcess): ForkProcess._start(self) def _future_done(self, task): - self.future.set_result(self.seconds) + if not self.future.cancelled(): + self.future.set_result(self.seconds) def _run(self): time.sleep(self.seconds)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: da5efb5e677d12c850ff02140e24a095a8efdafb Author: Zac Medico gentoo org> AuthorDate: Thu Apr 19 06:36:32 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 19 06:36:32 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=da5efb5e EventLoop.run_until_complete: remove unneeded partial Fixes: 25245d7eb86e ("EventLoop.run_until_complete: wait for done callbacks") pym/portage/util/_eventloop/EventLoop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 33fae26f2..6a8b906ed 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -818,7 +818,7 @@ class EventLoop(object): # is easily achieved by registering a done callback and waiting for # it to execute. waiter = self.create_future() - future.add_done_callback(functools.partial(waiter.set_result)) + future.add_done_callback(waiter.set_result) while not waiter.done(): self.iteration()
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 128c13ff8bd9a1685fb844f7a83469b2271f5e0f Author: Zac Medico gentoo org> AuthorDate: Thu Apr 19 06:19:00 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 19 06:25:32 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=128c13ff AsyncTaskFuture: override _wait method If the _wait method is called before the future is done, then wait for it to complete. pym/portage/util/_async/AsyncTaskFuture.py | 5 + 1 file changed, 5 insertions(+) diff --git a/pym/portage/util/_async/AsyncTaskFuture.py b/pym/portage/util/_async/AsyncTaskFuture.py index ee39183fe..c2316deeb 100644 --- a/pym/portage/util/_async/AsyncTaskFuture.py +++ b/pym/portage/util/_async/AsyncTaskFuture.py @@ -29,3 +29,8 @@ class AsyncTaskFuture(AsynchronousTask): else: self.returncode = 1 self.wait() + + def _wait(self): + if self.returncode is None: + self.scheduler.run_until_complete(self.future) + return self.returncode
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 25245d7eb86ed197b7d7cfead0dbe4ce8ad4bc5b Author: Zac Medico gentoo org> AuthorDate: Thu Apr 19 06:13:33 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 19 06:13:33 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=25245d7e EventLoop.run_until_complete: wait for done callbacks Since done callbacks are executed via call_soon, it's desirable to continue iterating until those callbacks have executed, which is easily achieved by registering a done callback and waiting for it to execute. pym/portage/util/_eventloop/EventLoop.py | 9 - 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 7208c3aa1..33fae26f2 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -812,7 +812,14 @@ class EventLoop(object): @raise: the Future's exception """ future = asyncio.ensure_future(future, loop=self._asyncio_wrapper) - while not future.done(): + + # Since done callbacks are executed via call_soon, it's desirable + # to continue iterating until those callbacks have executed, which + # is easily achieved by registering a done callback and waiting for + # it to execute. + waiter = self.create_future() + future.add_done_callback(functools.partial(waiter.set_result)) + while not waiter.done(): self.iteration() return future.result()
[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/, pym/portage/package/ebuild/
commit: cab78dba98c4309504e077c52a2ab0f0b7e27cc8 Author: Zac Medico gentoo org> AuthorDate: Tue Apr 17 20:56:49 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed Apr 18 21:55:58 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cab78dba emerge --usepkgonly: propagate implicit IUSE and USE_EXPAND (bug 640318) For emerge --usepkgonly mode, it's useful to be able to operate without dependence on a local ebuild repository and profile. Therefore, merge Packages header settings from the binary package database (including binhost(s) if enabled) into the profile configuration, in order to propagate implicit IUSE and USE_EXPAND settings for use with binary and installed packages. Values are appended, so the result is a union of elements. Also use ARCH from the binary package database if it's not defined by the profile, since ARCH is used for validation by emerge's profile_check function, and also for KEYWORDS logic in the _getmaskingstatus function. All changes are currently confined to --usepkgonly mode, since this is the mode where it is needed the most, and this ensures that behavior of source-based builds is completely unaffected. The changes only affect dependency calculations (where implicit IUSE plays a role) and the user interface (display of USE_EXPAND flags). The bash execution environment for binary and installed packages is completely unaffected, since that environment relies on variables loaded from environment.bz2 files. Bug: https://bugs.gentoo.org/640318 pym/_emerge/actions.py | 48 + pym/_emerge/main.py| 4 --- pym/portage/dbapi/bintree.py | 56 ++ pym/portage/package/ebuild/config.py | 7 ++-- pym/portage/package/ebuild/profile_iuse.py | 32 + 5 files changed, 127 insertions(+), 20 deletions(-) diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 347ef5fbc..b90aa8cb0 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -78,6 +78,7 @@ from _emerge.depgraph import backtrack_depgraph, depgraph, resume_depgraph from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange from _emerge.emergelog import emergelog from _emerge.is_valid_package_atom import is_valid_package_atom +from _emerge.main import profile_check from _emerge.MetadataRegen import MetadataRegen from _emerge.Package import Package from _emerge.ProgressHandler import ProgressHandler @@ -2209,9 +2210,21 @@ def action_uninstall(settings, trees, ldpath_mtimes, return rval def adjust_configs(myopts, trees): - for myroot in trees: + for myroot, mytrees in trees.items(): mysettings = trees[myroot]["vartree"].settings mysettings.unlock() + + # For --usepkgonly mode, propagate settings from the binary package + # database, so that it's possible to operate without dependence on + # a local ebuild repository and profile. + if ('--usepkgonly' in myopts and + mytrees['bintree']._propagate_config(mysettings)): + # Also propagate changes to the portdbapi doebuild_settings + # attribute which is used by Package instances for USE + # calculations (in support of --binpkg-respect-use). + mytrees['porttree'].dbapi.doebuild_settings = \ + portage.config(clone=mysettings) + adjust_config(myopts, mysettings) mysettings.lock() @@ -2868,7 +2881,27 @@ def run_action(emerge_config): "--usepkg", "--usepkgonly"): emerge_config.opts.pop(opt, None) + # Populate the bintree with current --getbinpkg setting. + # This needs to happen before: + # * expand_set_arguments, in case any sets use the bintree + # * adjust_configs and profile_check, in order to propagate settings + # implicit IUSE and USE_EXPAND settings from the binhost(s) + if (emerge_config.action in ('search', None) and + '--usepkg' in emerge_config.opts): + for mytrees in emerge_config.trees.values(): + try: + mytrees['bintree'].populate( + getbinpkgs='--getbinpkg' in emerge_config.opts) + except ParseError as e: + writemsg('\n\n!!!%s.\nSee make.conf(5) for more info.\n' +% (e,), noiselevel=-1) + return 1 + adjust_configs(emerge_config.opts, emerge_config.trees) + + if profile_check(emerge_config.trees, emerge_config.action) != os.EX_OK: + return 1 + apply_priorities(emerge_config.target_config.settings) if ("--