[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: a7c7af98d755f34e84d1f0f847e2c0d5cc5b7e2f
Author: Zac Medico gentoo org>
AuthorDate: Wed Jul 18 03:36:59 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Jul 18 03:40:05 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7c7af98
EventLoop: raise TypeError for unexpected call_* keyword args
pym/portage/util/_eventloop/EventLoop.py | 24
1 file changed, 24 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 084ff0c18..ffd12cff9 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -859,11 +859,23 @@ class EventLoop(object):
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
+ try:
+ unexpected = next(key for key in kwargs if key !=
'context')
+ except StopIteration:
+ pass
+ else:
+ raise TypeError("call_soon() got an unexpected keyword
argument '%s'" % unexpected)
return self._handle(self._idle_add(
self._call_soon_callback(callback, args)), self)
def call_soon_threadsafe(self, callback, *args, **kwargs):
"""Like call_soon(), but thread safe."""
+ try:
+ unexpected = next(key for key in kwargs if key !=
'context')
+ except StopIteration:
+ pass
+ else:
+ raise TypeError("call_soon_threadsafe() got an
unexpected keyword argument '%s'" % unexpected)
# idle_add provides thread safety
return self._handle(self.idle_add(
self._call_soon_callback(callback, args)), self)
@@ -909,6 +921,12 @@ class EventLoop(object):
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
+ try:
+ unexpected = next(key for key in kwargs if key !=
'context')
+ except StopIteration:
+ pass
+ else:
+ raise TypeError("call_later() got an unexpected keyword
argument '%s'" % unexpected)
return self._handle(self.timeout_add(
delay * 1000, self._call_soon_callback(callback,
args)), self)
@@ -936,6 +954,12 @@ class EventLoop(object):
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
+ try:
+ unexpected = next(key for key in kwargs if key !=
'context')
+ except StopIteration:
+ pass
+ else:
+ raise TypeError("call_at() got an unexpected keyword
argument '%s'" % unexpected)
delta = when - self.time()
return self.call_later(delta if delta > 0 else 0, callback,
*args)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: e46dd735cd4dde58cf3f8ef3cd2b8b29561f5b3e
Author: Zac Medico gentoo org>
AuthorDate: Tue Jul 17 19:27:28 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Jul 17 19:27:28 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e46dd735
EventLoop: use python2.7 compatible **kwargs for call_* context arg
Since python2.7 does not allow positional default arguments after
*args, use **kwargs instead.
Fixes: ae8cc32ccd81 ("EventLoop: add call_* context arg for python3.7 compat")
pym/portage/util/_eventloop/EventLoop.py | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 69ccbac2c..084ff0c18 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -832,7 +832,7 @@ class EventLoop(object):
return future.result()
- def call_soon(self, callback, *args, context=None):
+ def call_soon(self, callback, *args, **kwargs):
"""
Arrange for a callback to be called as soon as possible. The
callback
is called after call_soon() returns, when control returns to
the event
@@ -862,7 +862,7 @@ class EventLoop(object):
return self._handle(self._idle_add(
self._call_soon_callback(callback, args)), self)
- def call_soon_threadsafe(self, callback, *args, context=None):
+ def call_soon_threadsafe(self, callback, *args, **kwargs):
"""Like call_soon(), but thread safe."""
# idle_add provides thread safety
return self._handle(self.idle_add(
@@ -877,7 +877,7 @@ class EventLoop(object):
"""
return monotonic()
- def call_later(self, delay, callback, *args, context=None):
+ def call_later(self, delay, callback, *args, **kwargs):
"""
Arrange for the callback to be called after the given delay
seconds
(either an int or float).
@@ -912,7 +912,7 @@ class EventLoop(object):
return self._handle(self.timeout_add(
delay * 1000, self._call_soon_callback(callback,
args)), self)
- def call_at(self, when, callback, *args, context=None):
+ def call_at(self, when, callback, *args, **kwargs):
"""
Arrange for the callback to be called at the given absolute
timestamp when (an int or float), using the same time reference
as
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: ae8cc32ccd812661650647feffa1b10fc3ab5837 Author: Zac Medico gentoo org> AuthorDate: Tue Jul 17 19:04:28 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue Jul 17 19:05:38 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ae8cc32c EventLoop: add call_* context arg for python3.7 compat The context argument currently does nothing, but exists for minimal interoperability with Future instances that require it for PEP 567. pym/portage/util/_eventloop/EventLoop.py | 26 ++ 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index df76374d9..69ccbac2c 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -832,7 +832,7 @@ class EventLoop(object): return future.result() - def call_soon(self, callback, *args): + def call_soon(self, callback, *args, context=None): """ Arrange for a callback to be called as soon as possible. The callback is called after call_soon() returns, when control returns to the event @@ -844,18 +844,25 @@ class EventLoop(object): Any positional arguments after the callback will be passed to the callback when it is called. + The context argument currently does nothing, but exists for minimal + interoperability with Future instances that require it for PEP 567. + An object compatible with asyncio.Handle is returned, which can be used to cancel the callback. @type callback: callable @param callback: a function to call + @type context: contextvars.Context + @param context: An optional keyword-only context argument allows + specifying a custom contextvars.Context for the callback to run + in. The current context is used when no context is provided. @return: a handle which can be used to cancel the callback @rtype: asyncio.Handle (or compatible) """ return self._handle(self._idle_add( self._call_soon_callback(callback, args)), self) - def call_soon_threadsafe(self, callback, *args): + def call_soon_threadsafe(self, callback, *args, context=None): """Like call_soon(), but thread safe.""" # idle_add provides thread safety return self._handle(self.idle_add( @@ -870,7 +877,7 @@ class EventLoop(object): """ return monotonic() - def call_later(self, delay, callback, *args): + def call_later(self, delay, callback, *args, context=None): """ Arrange for the callback to be called after the given delay seconds (either an int or float). @@ -886,19 +893,26 @@ class EventLoop(object): it is called. If you want the callback to be called with some named arguments, use a closure or functools.partial(). + The context argument currently does nothing, but exists for minimal + interoperability with Future instances that require it for PEP 567. + Use functools.partial to pass keywords to the callback. @type delay: int or float @param delay: delay seconds @type callback: callable @param callback: a function to call + @type context: contextvars.Context + @param context: An optional keyword-only context argument allows + specifying a custom contextvars.Context for the callback to run + in. The current context is used when no context is provided. @return: a handle which can be used to cancel the callback @rtype: asyncio.Handle (or compatible) """ return self._handle(self.timeout_add( delay * 1000, self._call_soon_callback(callback, args)), self) - def call_at(self, when, callback, *args): + def call_at(self, when, callback, *args, context=None): """ Arrange for the callback to be called at the given absolute timestamp when (an int or float), using the same time reference as @@ -915,6 +929,10 @@ class EventLoop(object): @param when: absolute timestamp when to call callback @type callback: callable @param callback: a function to call + @type context: contextvars.Context + @param context: An optional keyword-only context argument allows + specifying a custom contextvars.Context for the callback to run +
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 6b50ba69f5a8e311fcddfb2e5c203631bd292c71 Author: Zac Medico gentoo org> AuthorDate: Thu Jun 21 21:03:38 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed Jun 27 03:04:38 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6b50ba69 AsyncioEventLoop: exit after unhandled exception (bug 658684) Fix portage commands to exit immediately for any unhandled exceptions that are raised while the asyncio event loop is running without a tty. If we have a tty then start the debugger, since in might aid in diagnosis of the problem. In order to avoid potential interference with API consumers, do not call set_exception_handler unless portage._internal_caller is True. Bug: https://bugs.gentoo.org/658684 pym/portage/util/_eventloop/asyncio_event_loop.py | 31 +++ 1 file changed, 31 insertions(+) diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py index c07b71103..ea0e03b23 100644 --- a/pym/portage/util/_eventloop/asyncio_event_loop.py +++ b/pym/portage/util/_eventloop/asyncio_event_loop.py @@ -2,7 +2,9 @@ # Distributed under the terms of the GNU General Public License v2 import os +import pdb import signal +import sys try: import asyncio as _real_asyncio @@ -53,6 +55,35 @@ class AsyncioEventLoop(_AbstractEventLoop): self.get_debug = loop.get_debug self._wakeup_fd = -1 + if portage._internal_caller: + loop.set_exception_handler(self._internal_caller_exception_handler) + + @staticmethod + def _internal_caller_exception_handler(loop, context): + """ + An exception handler which drops to a pdb shell if std* streams + refer to a tty, and otherwise kills the process with SIGTERM. + + In order to avoid potential interference with API consumers, this + implementation is only used when portage._internal_caller is True. + """ + loop.default_exception_handler(context) + if 'exception' in context: + # If we have a tty then start the debugger, since in might + # aid in diagnosis of the problem. If there's no tty, then + # exit immediately. + if all(s.isatty() for s in (sys.stdout, sys.stderr, sys.stdin)): + pdb.set_trace() + else: + # Normally emerge will wait for all coroutines to complete + # after SIGTERM has been received. However, an unhandled + # exception will prevent the interrupted coroutine from + # completing, therefore use the default SIGTERM handler + # in order to ensure that emerge exits immediately (though + # uncleanly). + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGTERM) + def _create_future(self): """ Provide AbstractEventLoop.create_future() for python3.4.
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 4fb5ef2ce2cb27ae155a25bfa5a4666597afb6ac
Author: Zac Medico gentoo org>
AuthorDate: Sat May 26 20:20:06 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Sat May 26 20:20:06 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4fb5ef2c
EventLoop.close: fix 'AttributeError: close' for python2
For python2 without epoll, fix handling of missing
'close' attribute on self._poll_obj.
Fixes: 4095be74985c ("Add ForkExecutor (bug 649588)")
Reported-by: Brian Evans gentoo.org>
pym/portage/util/_eventloop/EventLoop.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index de0795224..df76374d9 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -968,7 +968,7 @@ class EventLoop(object):
executor.shutdown(wait=True)
if self._poll_obj is not None:
- close = getattr(self._poll_obj, 'close')
+ close = getattr(self._poll_obj, 'close', None)
if close is not None:
close()
self._poll_obj = None
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
commit: 466cc6c60fcf1d56049af7a1d37fc959f6808351
Author: Zac Medico gentoo org>
AuthorDate: Fri May 25 16:07:33 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Fri May 25 16:10:34 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=466cc6c6
SonameDepsProcessor: fix 'set' object has no attribute 'items' (bug 656492)
Use collections.defaultdict to ensure that requires_map
contains correct defaults.
Fixes: 1364cd44e7a6 ("SonameDepsProcessor: handle internal libs without
DT_SONAME (bug 646190)")
Bug: https://bugs.gentoo.org/656492
pym/portage/util/_dyn_libs/soname_deps.py | 9 +
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/soname_deps.py
b/pym/portage/util/_dyn_libs/soname_deps.py
index c6302afc2..544cbc8f1 100644
--- a/pym/portage/util/_dyn_libs/soname_deps.py
+++ b/pym/portage/util/_dyn_libs/soname_deps.py
@@ -3,7 +3,9 @@
from __future__ import unicode_literals
+import collections
import fnmatch
+import functools
from itertools import chain
import os
import re
@@ -33,7 +35,8 @@ class SonameDepsProcessor(object):
"""
self._provides_exclude = self._exclude_pattern(provides_exclude)
self._requires_exclude = self._exclude_pattern(requires_exclude)
- self._requires_map = {}
+ self._requires_map = collections.defaultdict(
+ functools.partial(collections.defaultdict, set))
self._provides_map = {}
self._provides_unfiltered = {}
self._basename_map = {}
@@ -84,8 +87,7 @@ class SonameDepsProcessor(object):
for x in entry.needed:
if (self._requires_exclude is None or
self._requires_exclude.match(x) is
None):
- self._requires_map.setdefault(
- multilib_cat, {}).setdefault(x,
set()).add(runpaths)
+
self._requires_map[multilib_cat][x].add(runpaths)
if entry.soname:
self._provides_unfiltered.setdefault(
@@ -105,7 +107,6 @@ class SonameDepsProcessor(object):
provides_unfiltered = self._provides_unfiltered
for multilib_cat in set(chain(requires_map, provides_map)):
- requires_map.setdefault(multilib_cat, set())
provides_map.setdefault(multilib_cat, set())
provides_unfiltered.setdefault(multilib_cat, set())
for soname, consumers in
list(requires_map[multilib_cat].items()):
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 8a8f527c7587bf388645ad703e0305797a26c3b4 Author: Zac Medico gentoo org> AuthorDate: Fri May 25 03:06:08 2018 + Commit: Zac Medico gentoo org> CommitDate: Fri May 25 03:06:32 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8a8f527c AsyncioEventLoop: remove redundant set_wakeup_fd call pym/portage/util/_eventloop/asyncio_event_loop.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py index 65b354544..c07b71103 100644 --- a/pym/portage/util/_eventloop/asyncio_event_loop.py +++ b/pym/portage/util/_eventloop/asyncio_event_loop.py @@ -104,5 +104,3 @@ class AsyncioEventLoop(_AbstractEventLoop): return self._loop.run_until_complete(future) finally: self._wakeup_fd = signal.set_wakeup_fd(-1) - if self._wakeup_fd != -1: - signal.set_wakeup_fd(-1)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/, pym/portage/tests/util/dyn_libs/
commit: 1364cd44e7a6232bf425c4573b5bd3d6738d49a4
Author: Zac Medico gentoo org>
AuthorDate: Sun May 20 06:40:19 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Thu May 24 20:31:02 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1364cd44
SonameDepsProcessor: handle internal libs without DT_SONAME (bug 646190)
Packages like ebtables have internal libraries that lack a DT_SONAME
field in their ELF header. Consumers of these internal libraries have
DT_RUNPATH entries that refer to the directory containing the internal
libraries. For library dependencies that are satisfied by internal
libraries like this, it is inappropriate for SonameDepsProcessor to
include these depenedencies in the REQUIRES metadata, therefore fix
SonameDepsProcessor to automatically detect this case and exclude
these dependencies from the REQUIRES metadata. This solves incorrect
reporting of broken soname dependencies like the following:
$ emerge -p --depclean --ignore-soname-deps=n
Calculating dependencies... done!
* Broken soname dependencies found:
*
* x86_64: libebt_redirect.so required by:
* net-firewall/ebtables-2.0.10.4
*
* x86_64: libebt_log.so required by:
* net-firewall/ebtables-2.0.10.4
Bug: https://bugs.gentoo.org/646190
pym/portage/tests/util/dyn_libs/__init__.py| 0
pym/portage/tests/util/dyn_libs/__test__.py| 0
.../tests/util/dyn_libs/test_soname_deps.py| 34 +
pym/portage/util/_dyn_libs/soname_deps.py | 35 --
4 files changed, 66 insertions(+), 3 deletions(-)
diff --git a/pym/portage/tests/util/dyn_libs/__init__.py
b/pym/portage/tests/util/dyn_libs/__init__.py
new file mode 100644
index 0..e69de29bb
diff --git a/pym/portage/tests/util/dyn_libs/__test__.py
b/pym/portage/tests/util/dyn_libs/__test__.py
new file mode 100644
index 0..e69de29bb
diff --git a/pym/portage/tests/util/dyn_libs/test_soname_deps.py
b/pym/portage/tests/util/dyn_libs/test_soname_deps.py
new file mode 100644
index 0..823890c91
--- /dev/null
+++ b/pym/portage/tests/util/dyn_libs/test_soname_deps.py
@@ -0,0 +1,34 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.util._dyn_libs.NeededEntry import NeededEntry
+from portage.util._dyn_libs.soname_deps import SonameDepsProcessor
+
+
+class SonameDepsProcessorTestCase(TestCase):
+
+ def testInternalLibsWithoutSoname(self):
+ """
+ Test handling of internal libraries that lack an soname, which
are
+ resolved via DT_RUNPATH, see ebtables for example (bug 646190).
+ """
+ needed_elf_2 = """
+X86_64;/sbin/ebtables;;/lib64/ebtables;libebt_802_3.so,libebtable_broute.so,libc.so.6;x86_64
+X86_64;/lib64/ebtables/libebtable_broute.so;;;libc.so.6;x86_64
+X86_64;/lib64/ebtables/libebt_802_3.so;;;libc.so.6;x86_64
+"""
+ soname_deps = SonameDepsProcessor('', '')
+
+ for line in needed_elf_2.splitlines():
+ if not line:
+ continue
+ entry = NeededEntry.parse(None, line)
+ soname_deps.add(entry)
+
+ self.assertEqual(soname_deps.provides, None)
+ # Prior to the fix for bug 646190, REQUIRES contained
references to
+ # the internal libebt* libraries which are resolved via a
DT_RUNPATH
+ # entry referring to the /lib64/ebtables directory that
contains the
+ # internal libraries.
+ self.assertEqual(soname_deps.requires, 'x86_64: libc.so.6\n')
diff --git a/pym/portage/util/_dyn_libs/soname_deps.py
b/pym/portage/util/_dyn_libs/soname_deps.py
index a7d595429..c6302afc2 100644
--- a/pym/portage/util/_dyn_libs/soname_deps.py
+++ b/pym/portage/util/_dyn_libs/soname_deps.py
@@ -9,6 +9,11 @@ import os
import re
from portage.util import shlex_split
+from portage.util import (
+ normalize_path,
+ varexpand,
+)
+
class SonameDepsProcessor(object):
"""
@@ -31,6 +36,7 @@ class SonameDepsProcessor(object):
self._requires_map = {}
self._provides_map = {}
self._provides_unfiltered = {}
+ self._basename_map = {}
self._provides = None
self._requires = None
self._intersected = False
@@ -62,15 +68,24 @@ class SonameDepsProcessor(object):
raise AssertionError(
"Missing multilib category data: %s" %
entry.filename)
+ self._basename_map.setdefault(
+ os.path.basename(entry.filename), []).append(entry)
+
if entry.needed and (
self._requires_exclude is None or
self._requires_exclude.match(
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: b28b01a78e4cc0199111e804af5b56a33319356d Author: Zac Medico gentoo org> AuthorDate: Mon May 14 15:33:15 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon May 14 15:35:53 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b28b01a7 ForkProcess: unregister SIGCHLD and wakeup_fd (bug 655656) In order to prevent forked processes from invoking the parent process's SIGCHLD handler and writing to wakeup_fd (triggering BlockingIOError), unregister the SIGCHLD and wakeup_fd. Bug: https://bugs.gentoo.org/655656 Reported-by: Helmut Jarausch igpm.rwth-aachen.de> pym/portage/util/_async/ForkProcess.py | 10 ++ 1 file changed, 10 insertions(+) diff --git a/pym/portage/util/_async/ForkProcess.py b/pym/portage/util/_async/ForkProcess.py index 25f72d308..d84e93833 100644 --- a/pym/portage/util/_async/ForkProcess.py +++ b/pym/portage/util/_async/ForkProcess.py @@ -37,6 +37,16 @@ class ForkProcess(SpawnProcess): signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) + # Unregister SIGCHLD handler and wakeup_fd for the parent + # process's event loop (bug 655656). + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + try: + wakeup_fd = signal.set_wakeup_fd(-1) + if wakeup_fd > 0: + os.close(wakeup_fd) + except (ValueError, OSError): + pass + portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring).
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 65379f436759dfbc4d56e52f1a145950779ebb60 Author: Zac Medico gentoo org> AuthorDate: Sun May 13 16:45:27 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun May 13 16:57:38 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=65379f43 AsyncioEventLoop: suppress BlockingIOError warning during loop close (bug 655656) Disable the asyncio event loop's SIGCHLD handler before attempting to close it, in order to suppress a harmless BlockingIOError warning during loop close. Closes: https://bugs.gentoo.org/655656 Reported-by: Helmut Jarausch igpm.rwth-aachen.de> pym/portage/util/_eventloop/asyncio_event_loop.py | 10 +- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py index b365939b0..bf5937de8 100644 --- a/pym/portage/util/_eventloop/asyncio_event_loop.py +++ b/pym/portage/util/_eventloop/asyncio_event_loop.py @@ -1,6 +1,8 @@ # Copyright 2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import signal + try: import asyncio as _real_asyncio from asyncio.events import AbstractEventLoop as _AbstractEventLoop @@ -31,7 +33,6 @@ class AsyncioEventLoop(_AbstractEventLoop): self.call_at = loop.call_at self.is_running = loop.is_running self.is_closed = loop.is_closed - self.close = loop.close self.create_future = (loop.create_future if hasattr(loop, 'create_future') else self._create_future) self.create_task = loop.create_task @@ -75,3 +76,10 @@ class AsyncioEventLoop(_AbstractEventLoop): @return: the internal event loop's AbstractEventLoop interface """ return self + + def close(self): + # Suppress spurious error messages like the following for bug 655656: + # Exception ignored when trying to write to the signal wakeup fd: + # BlockingIOError: [Errno 11] Resource temporarily unavailable + self._loop.remove_signal_handler(signal.SIGCHLD) + self._loop.close()
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/_asyncio/, pym/portage/util/futures/
commit: 5e628787e6f4c720680aeeb8beeac88e37988a9e
Author: Zac Medico gentoo org>
AuthorDate: Wed May 9 07:33:37 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Wed May 9 07:40:34 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5e628787
DefaultEventLoopPolicy: raise NotImplementedError, not RecursionError
Since the DefaultEventLoopPolicy wraps the underlying asyncio event
loop policy, raise NotImplementedError if the current instance is set
as the underlying event loop policy. This avoids a RecursionError
that would flood the terminal with a large stack trace.
pym/portage/util/futures/_asyncio/__init__.py | 6 +++---
pym/portage/util/futures/unix_events.py | 23 ++-
2 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/futures/_asyncio/__init__.py
b/pym/portage/util/futures/_asyncio/__init__.py
index 940da4762..acfd59396 100644
--- a/pym/portage/util/futures/_asyncio/__init__.py
+++ b/pym/portage/util/futures/_asyncio/__init__.py
@@ -32,7 +32,7 @@ except ImportError:
import portage
portage.proxy.lazyimport.lazyimport(globals(),
- 'portage.util.futures.unix_events:DefaultEventLoopPolicy',
+ 'portage.util.futures.unix_events:_PortageEventLoopPolicy',
)
from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as
_AsyncioEventLoop
from portage.util._eventloop.global_event_loop import (
@@ -67,7 +67,7 @@ def get_event_loop_policy():
global _lock, _policy
with _lock:
if _policy is None:
- _policy = DefaultEventLoopPolicy()
+ _policy = _PortageEventLoopPolicy()
return _policy
@@ -81,7 +81,7 @@ def set_event_loop_policy(policy):
"""
global _lock, _policy
with _lock:
- _policy = policy or DefaultEventLoopPolicy()
+ _policy = policy or _PortageEventLoopPolicy()
def get_event_loop():
diff --git a/pym/portage/util/futures/unix_events.py
b/pym/portage/util/futures/unix_events.py
index 8eb369f8b..3381eaa7d 100644
--- a/pym/portage/util/futures/unix_events.py
+++ b/pym/portage/util/futures/unix_events.py
@@ -681,4 +681,25 @@ class
_PortageEventLoopPolicy(events.AbstractEventLoopPolicy):
return _global_event_loop()._asyncio_child_watcher
-DefaultEventLoopPolicy = _PortageEventLoopPolicy
+class _AsyncioEventLoopPolicy(_PortageEventLoopPolicy):
+ """
+ A subclass of _PortageEventLoopPolicy which raises
+ NotImplementedError if it is set as the real asyncio event loop
+ policy, since this class is intended to *wrap* the real asyncio
+ event loop policy.
+ """
+ def _check_recursion(self):
+ if _real_asyncio.get_event_loop_policy() is self:
+ raise NotImplementedError('this class is only a
wrapper')
+
+ def get_event_loop(self):
+ self._check_recursion()
+ return super(_AsyncioEventLoopPolicy, self).get_event_loop()
+
+ def get_child_watcher(self):
+ self._check_recursion()
+ return super(_AsyncioEventLoopPolicy, self).get_child_watcher()
+
+
+DefaultEventLoopPolicy = (_AsyncioEventLoopPolicy if _asyncio_enabled
+ else _PortageEventLoopPolicy)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/_asyncio/, pym/portage/util/futures/
commit: 920b90fd0883dbd36f0290d08c9af49a208c2950
Author: Zac Medico gentoo org>
AuthorDate: Wed May 9 04:59:59 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Wed May 9 04:59:59 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=920b90fd
_wrap_loop: default to global_event_loop behavior
The default loop returned by _wrap_loop should be consistent
with global_event_loop, in order to avoid accidental registration
of callbacks with a loop that is not intended to run.
Fixes 96cc07326391 ("global_event_loop: use asyncio event loop (bug 654390)")
pym/portage/util/futures/_asyncio/__init__.py | 14 ++
pym/portage/util/futures/unix_events.py | 40 +--
2 files changed, 10 insertions(+), 44 deletions(-)
diff --git a/pym/portage/util/futures/_asyncio/__init__.py
b/pym/portage/util/futures/_asyncio/__init__.py
index 1273afa02..940da4762 100644
--- a/pym/portage/util/futures/_asyncio/__init__.py
+++ b/pym/portage/util/futures/_asyncio/__init__.py
@@ -35,7 +35,10 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures.unix_events:DefaultEventLoopPolicy',
)
from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as
_AsyncioEventLoop
-from portage.util._eventloop.global_event_loop import _asyncio_enabled
+from portage.util._eventloop.global_event_loop import (
+ _asyncio_enabled,
+ global_event_loop as _global_event_loop,
+)
from portage.util.futures.futures import (
CancelledError,
Future,
@@ -168,14 +171,15 @@ def _wrap_loop(loop=None):
@rtype: asyncio.AbstractEventLoop (or compatible)
@return: event loop
"""
- return loop or get_event_loop()
+ return loop or _global_event_loop()
if _asyncio_enabled:
- get_event_loop_policy = _real_asyncio.get_event_loop_policy
- set_event_loop_policy = _real_asyncio.set_event_loop_policy
+ # The default loop returned by _wrap_loop should be consistent
+ # with global_event_loop, in order to avoid accidental registration
+ # of callbacks with a loop that is not intended to run.
def _wrap_loop(loop=None):
- loop = loop or get_event_loop()
+ loop = loop or _global_event_loop()
return (loop if hasattr(loop, '_asyncio_wrapper')
else _AsyncioEventLoop(loop=loop))
diff --git a/pym/portage/util/futures/unix_events.py
b/pym/portage/util/futures/unix_events.py
index ce520db00..8eb369f8b 100644
--- a/pym/portage/util/futures/unix_events.py
+++ b/pym/portage/util/futures/unix_events.py
@@ -681,42 +681,4 @@ class
_PortageEventLoopPolicy(events.AbstractEventLoopPolicy):
return _global_event_loop()._asyncio_child_watcher
-class _AsyncioEventLoopPolicy(_PortageEventLoopPolicy):
- """
- Implementation of asyncio.AbstractEventLoopPolicy based on asyncio's
- event loop. This supports running event loops in forks,
- which is not supported by the default asyncio event loop policy,
- see https://bugs.python.org/issue22087 and also
- https://bugs.python.org/issue29703 which affects pypy3-5.10.1.
- """
- _MAIN_PID = os.getpid()
-
- def __init__(self):
- super(_AsyncioEventLoopPolicy, self).__init__()
- self._default_policy = _real_asyncio.DefaultEventLoopPolicy()
-
- def get_event_loop(self):
- """
- Get the event loop for the current context.
-
- Returns an event loop object implementing the AbstractEventLoop
- interface.
-
- @rtype: asyncio.AbstractEventLoop (or compatible)
- @return: the current event loop policy
- """
- if os.getpid() == self._MAIN_PID:
- return self._default_policy.get_event_loop()
- else:
- return super(_AsyncioEventLoopPolicy,
self).get_event_loop()
-
- def get_child_watcher(self):
- """Get the watcher for child processes."""
- if os.getpid() == self._MAIN_PID:
- return self._default_policy.get_child_watcher()
- else:
- return super(_AsyncioEventLoopPolicy,
self).get_child_watcher()
-
-
-DefaultEventLoopPolicy = (_AsyncioEventLoopPolicy if _asyncio_enabled
- else _PortageEventLoopPolicy)
+DefaultEventLoopPolicy = _PortageEventLoopPolicy
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, pym/portage/util/_eventloop/, ...
commit: 96cc073263910e7c1b0f48cc08a4db4b56ffe408
Author: Zac Medico gentoo org>
AuthorDate: Tue May 1 08:15:39 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Tue May 8 16:42:03 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=96cc0732
global_event_loop: use asyncio event loop (bug 654390)
For python3.4 and later, in the main process, replace portage's
internal event loop with the standard library's asyncio event
loop. Continue to use portage's internal event loop in subprocesses,
since asyncio's event loop is not guaranteed to work well in
subprocesses (see upstream python issues 22087 and 29703).
An _AsyncioEventLoopPolicy class, derived from _PortageEventLoopPolicy,
is needed for some unit tests that modify asyncio's event loop policy.
This policy is not needed for anything other than unit testing. Portage
uses asyncio's default event loop policy, and API consumers are free to
use any desired event loop policy.
Portage's asynchronous functions that accept a 'loop' parameter will
work with any compatible asyncio.AbstractEventLoop implementation, since
an internal _wrap_loop function automatically adapts the loop for
internal use.
Bug: https://bugs.gentoo.org/654390
Reviewed-by: Brian Dolbec gentoo.org>
Reviewed-by: Alec Warner gentoo.org>
pym/portage/util/_async/SchedulerInterface.py | 3 +
pym/portage/util/_eventloop/asyncio_event_loop.py | 77 +++
pym/portage/util/_eventloop/global_event_loop.py | 5 +-
pym/portage/util/futures/_asyncio/__init__.py | 17 +
pym/portage/util/futures/unix_events.py | 43 -
5 files changed, 143 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_async/SchedulerInterface.py
b/pym/portage/util/_async/SchedulerInterface.py
index f1a3e9b0b..ec6417da1 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -33,6 +33,9 @@ class SchedulerInterface(SlotObject):
"time",
"_asyncio_child_watcher",
+ # This attribute it used by _wrap_loop to detect if the
+ # loop already has a suitable wrapper.
+ "_asyncio_wrapper",
)
__slots__ = _event_loop_attrs + ("_event_loop", "_is_background")
diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py
b/pym/portage/util/_eventloop/asyncio_event_loop.py
new file mode 100644
index 0..b365939b0
--- /dev/null
+++ b/pym/portage/util/_eventloop/asyncio_event_loop.py
@@ -0,0 +1,77 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+try:
+ import asyncio as _real_asyncio
+ from asyncio.events import AbstractEventLoop as _AbstractEventLoop
+except ImportError:
+ # Allow ImportModulesTestCase to succeed.
+ _real_asyncio = None
+ _AbstractEventLoop = object
+
+
+class AsyncioEventLoop(_AbstractEventLoop):
+ """
+ Implementation of asyncio.AbstractEventLoop which wraps asyncio's
+ event loop and is minimally compatible with _PortageEventLoop.
+ """
+
+ # Use portage's internal event loop in subprocesses, as a workaround
+ # for https://bugs.python.org/issue22087, and also
+ # https://bugs.python.org/issue29703 which affects pypy3-5.10.1.
+ supports_multiprocessing = False
+
+ def __init__(self, loop=None):
+ loop = loop or _real_asyncio.get_event_loop()
+ self._loop = loop
+ self.run_until_complete = loop.run_until_complete
+ self.call_soon = loop.call_soon
+ self.call_soon_threadsafe = loop.call_soon_threadsafe
+ self.call_later = loop.call_later
+ self.call_at = loop.call_at
+ self.is_running = loop.is_running
+ self.is_closed = loop.is_closed
+ self.close = loop.close
+ self.create_future = (loop.create_future
+ if hasattr(loop, 'create_future') else
self._create_future)
+ self.create_task = loop.create_task
+ self.add_reader = loop.add_reader
+ self.remove_reader = loop.remove_reader
+ self.add_writer = loop.add_writer
+ self.remove_writer = loop.remove_writer
+ self.run_in_executor = loop.run_in_executor
+ self.time = loop.time
+ self.default_exception_handler = loop.default_exception_handler
+ self.call_exception_handler = loop.call_exception_handler
+ self.set_debug = loop.set_debug
+ self.get_debug = loop.get_debug
+
+ def _create_future(self):
+ """
+ Provide AbstractEventLoop.create_future() for python3.4.
+ """
+ return _real_asyncio.Future(loop=self._loop)
+
+ @property
+ def _asyncio_child_watcher(self):
+
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/, pym/portage/tests/util/futures/, ...
commit: b3b15c451cc21a2c53638f0eacc8396e395dcab3 Author: Zac Medico gentoo org> AuthorDate: Mon May 7 06:24:22 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon May 7 06:28:47 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b3b15c45 retry: add loop parameter during decoration pym/portage/sync/modules/rsync/rsync.py | 2 +- pym/portage/tests/util/futures/test_retry.py | 16 pym/portage/util/futures/retry.py| 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py index 1b8941ff6..070798a53 100644 --- a/pym/portage/sync/modules/rsync/rsync.py +++ b/pym/portage/sync/modules/rsync/rsync.py @@ -173,7 +173,7 @@ class RsyncSync(NewBase): loop = global_event_loop() func_coroutine = functools.partial(loop.run_in_executor, None, noisy_refresh_keys) - decorated_func = retry_decorator(func_coroutine) + decorated_func = retry_decorator(func_coroutine, loop=loop) loop.run_until_complete(decorated_func()) out.eend(0) except (GematoException, asyncio.TimeoutError) as e: diff --git a/pym/portage/tests/util/futures/test_retry.py b/pym/portage/tests/util/futures/test_retry.py index 16ecccbc7..7a1e76280 100644 --- a/pym/portage/tests/util/futures/test_retry.py +++ b/pym/portage/tests/util/futures/test_retry.py @@ -85,7 +85,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wrap_coroutine_func(SucceedLater(1)) decorator = retry(try_max=, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine) + decorated_func = decorator(func_coroutine, loop=loop) result = loop.run_until_complete(decorated_func()) self.assertEqual(result, 'success') @@ -94,7 +94,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wrap_coroutine_func(SucceedNever()) decorator = retry(try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine) + decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception().__cause__, SucceedNeverException)) @@ -104,7 +104,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wrap_coroutine_func(SucceedNever()) decorator = retry(reraise=True, try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine) + decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception(), SucceedNeverException)) @@ -114,7 +114,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wrap_coroutine_func(HangForever()) decorator = retry(try_max=2, try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine) + decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception().__cause__, asyncio.TimeoutError)) @@ -124,7 +124,7 @@ class RetryTestCase(TestCase): func_coroutine = self._wrap_coroutine_func(HangForever()) decorator = retry(reraise=True, try_max=2, try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine) + decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception(), asyncio.TimeoutError)) @@ -134,7 +134,7 @@ class RetryTestCase(TestCase): func_coroutine =
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: a1360e017b7ee8156ad4ad850e2f8ea40228ca1a Author: Zac Medico gentoo org> AuthorDate: Mon May 7 00:06:07 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon May 7 00:19:54 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a1360e01 EventLoop.run_in_executor: use asyncio.wrap_future Since executors from the concurrent.futures package return concurrent.futures.Future, it's necessary to wrap them. pym/portage/util/_eventloop/EventLoop.py | 11 ++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index fc7380b03..de0795224 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -13,6 +13,11 @@ import signal import sys import traceback +try: + import asyncio as _real_asyncio +except ImportError: + _real_asyncio = None + try: import fcntl except ImportError: @@ -937,7 +942,11 @@ class EventLoop(object): if executor is None: executor = ForkExecutor(loop=self) self._default_executor = executor - return executor.submit(func, *args) + future = executor.submit(func, *args) + if _real_asyncio is not None: + future = _real_asyncio.wrap_future(future, + loop=self._asyncio_wrapper) + return future def is_running(self): """Return whether the event loop is currently running."""
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/_asyncio/, pym/portage/tests/util/futures/asyncio/, ...
commit: 85ac23b7c0c58cef72d22281d66d086521c01e3e
Author: Zac Medico gentoo org>
AuthorDate: Sun May 6 11:05:03 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Sun May 6 11:41:45 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=85ac23b7
asyncio: add _wrap_loop helper (bug 654390)
In order to deal with asyncio event loop compatibility issues, add
a _wrap_loop helper. For example, since python3.4 does not have the
AbstractEventLoop.create_future() method, this helper function can
be used to add a wrapper that implements the create_future method
for python3.4.
Bug: https://bugs.gentoo.org/654390
pym/portage/dbapi/porttree.py | 12 ++--
.../tests/util/futures/asyncio/test_child_watcher.py | 2 +-
.../util/futures/asyncio/test_event_loop_in_fork.py | 8
.../tests/util/futures/asyncio/test_pipe_closed.py| 4 ++--
.../util/futures/asyncio/test_run_until_complete.py | 2 +-
.../util/futures/asyncio/test_subprocess_exec.py | 4 ++--
pym/portage/util/futures/_asyncio/__init__.py | 19 ++-
pym/portage/util/futures/_asyncio/tasks.py| 7 +--
pym/portage/util/futures/executor/fork.py | 4 ++--
pym/portage/util/futures/iter_completed.py| 7 +++
pym/portage/util/futures/retry.py | 3 +--
11 files changed, 45 insertions(+), 27 deletions(-)
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 801b5658a..3e36024ff 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -36,7 +36,7 @@ from portage import _encodings
from portage import _unicode_encode
from portage import OrderedDict
from portage.util._eventloop.EventLoop import EventLoop
-from portage.util._eventloop.global_event_loop import global_event_loop
+from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
@@ -325,8 +325,8 @@ class portdbapi(dbapi):
@property
def _event_loop(self):
if portage._internal_caller:
- # For internal portage usage, the global_event_loop is
safe.
- return global_event_loop()
+ # For internal portage usage, asyncio._wrap_loop() is
safe.
+ return asyncio._wrap_loop()
else:
# For external API consumers, use a local EventLoop,
since
# we don't want to assume that it's safe to override the
@@ -611,7 +611,7 @@ class portdbapi(dbapi):
# to simultaneous instantiation of multiple event loops here.
# Callers of this method certainly want the same event loop to
# be used for all calls.
- loop = loop or global_event_loop()
+ loop = asyncio._wrap_loop(loop)
future = loop.create_future()
cache_me = False
if myrepo is not None:
@@ -751,7 +751,7 @@ class portdbapi(dbapi):
a set of alternative URIs.
@rtype: asyncio.Future (or compatible)
"""
- loop = loop or global_event_loop()
+ loop = asyncio._wrap_loop(loop)
result = loop.create_future()
def aux_get_done(aux_get_future):
@@ -1419,7 +1419,7 @@ def _async_manifest_fetchlist(portdb, repo_config, cp,
cpv_list=None,
@return: a Future resulting in a Mapping compatible with FetchlistDict
@rtype: asyncio.Future (or compatible)
"""
- loop = loop or global_event_loop()
+ loop = asyncio._wrap_loop(loop)
result = loop.create_future()
cpv_list = (portdb.cp_list(cp, mytree=repo_config.location)
if cpv_list is None else cpv_list)
diff --git a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
index dca01be56..8ef497544 100644
--- a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
+++ b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
@@ -28,7 +28,7 @@ class ChildWatcherTestCase(TestCase):
args_tuple = ('hello', 'world')
- loop = asyncio.get_event_loop()
+ loop = asyncio._wrap_loop()
future = loop.create_future()
def callback(pid, returncode, *args):
diff --git a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py
b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py
index 7868d792a..19588bf3a 100644
--- a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py
+++ b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py
@@ -11,14 +11,14 @@ from portage.util.futures.unix_events import
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 2495fe6ff060e2ed8ee54e08a4dec132de1f4984
Author: Zac Medico gentoo org>
AuthorDate: Mon Apr 30 05:52:07 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Apr 30 06:20:01 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2495fe6f
PipeLogger: add_reader asyncio compat (bug 654382)
Use add_reader for asyncio compatibility.
Bug: https://bugs.gentoo.org/654382
pym/portage/util/_async/PipeLogger.py | 20 ++--
1 file changed, 6 insertions(+), 14 deletions(-)
diff --git a/pym/portage/util/_async/PipeLogger.py
b/pym/portage/util/_async/PipeLogger.py
index 02de74b16..61000271d 100644
--- a/pym/portage/util/_async/PipeLogger.py
+++ b/pym/portage/util/_async/PipeLogger.py
@@ -21,7 +21,7 @@ class PipeLogger(AbstractPollTask):
"""
__slots__ = ("input_fd", "log_file_path", "stdout_fd") + \
- ("_log_file", "_log_file_real", "_reg_id")
+ ("_log_file", "_log_file_real")
def _start(self):
@@ -57,8 +57,7 @@ class PipeLogger(AbstractPollTask):
fcntl.fcntl(fd, fcntl.F_SETFD,
fcntl.fcntl(fd, fcntl.F_GETFD) |
fcntl.FD_CLOEXEC)
- self._reg_id = self.scheduler.io_add_watch(fd,
- self._registered_events, self._output_handler)
+ self.scheduler.add_reader(fd, self._output_handler, fd)
self._registered = True
def _cancel(self):
@@ -66,14 +65,14 @@ class PipeLogger(AbstractPollTask):
if self.returncode is None:
self.returncode = self._cancelled_returncode
- def _output_handler(self, fd, event):
+ def _output_handler(self, fd):
background = self.background
stdout_fd = self.stdout_fd
log_file = self._log_file
while True:
- buf = self._read_buf(fd, event)
+ buf = self._read_buf(fd, None)
if buf is None:
# not a POLLIN event, EAGAIN, etc...
@@ -124,20 +123,13 @@ class PipeLogger(AbstractPollTask):
log_file.write(buf)
log_file.flush()
- self._unregister_if_appropriate(event)
-
- return True
-
def _unregister(self):
-
- if self._reg_id is not None:
- self.scheduler.source_remove(self._reg_id)
- self._reg_id = None
-
if self.input_fd is not None:
if isinstance(self.input_fd, int):
+ self.scheduler.remove_reader(self.input_fd)
os.close(self.input_fd)
else:
+
self.scheduler.remove_reader(self.input_fd.fileno())
self.input_fd.close()
self.input_fd = None
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: abf4b7abcdc53bbbd86eb6f79d45417b54ed64d2 Author: Zac Medico gentoo org> AuthorDate: Sun Apr 29 22:12:13 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 29 22:18:28 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=abf4b7ab AsyncScheduler: call_later asyncio compat (bug 591760) Use call_later for asyncio compatibility. Also remove the timeout_add method from SchedulerInterface, since there are no more consumers. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/AsyncScheduler.py | 9 +++-- pym/portage/util/_async/SchedulerInterface.py | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py index 90e803eba..c6b523eaa 100644 --- a/pym/portage/util/_async/AsyncScheduler.py +++ b/pym/portage/util/_async/AsyncScheduler.py @@ -65,6 +65,11 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): task.addExitListener(self._task_exit) task.start() + if self._loadavg_check_id is not None: + self._loadavg_check_id.cancel() + self._loadavg_check_id = self._event_loop.call_later( + self._loadavg_latency, self._schedule) + # Triggers cleanup and exit listeners if there's nothing left to do. self.poll() @@ -80,14 +85,14 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): (self._max_jobs is True or self._max_jobs > 1): # We have to schedule periodically, in case the load # average has changed since the last call. - self._loadavg_check_id = self._event_loop.timeout_add( + self._loadavg_check_id = self._event_loop.call_later( self._loadavg_latency, self._schedule) self._schedule() def _cleanup(self): super(AsyncScheduler, self)._cleanup() if self._loadavg_check_id is not None: - self._event_loop.source_remove(self._loadavg_check_id) + self._loadavg_check_id.cancel() self._loadavg_check_id = None def _async_wait(self): diff --git a/pym/portage/util/_async/SchedulerInterface.py b/pym/portage/util/_async/SchedulerInterface.py index ff39bc587..56b844616 100644 --- a/pym/portage/util/_async/SchedulerInterface.py +++ b/pym/portage/util/_async/SchedulerInterface.py @@ -15,7 +15,6 @@ class SchedulerInterface(SlotObject): "IO_NVAL", "IO_OUT", "IO_PRI", "io_add_watch", "source_remove", - "timeout_add", "add_reader", "add_writer",
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: be800bf0153a28ce034277d103a2021f93ac8b2e Author: Zac Medico gentoo org> AuthorDate: Sun Apr 29 05:10:27 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 29 05:58:58 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=be800bf0 PopenProcess: add_child_handler asyncio compat (bug 591760) Migrate to asyncio's AbstractChildWatcher.add_child_handler interface. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/PopenProcess.py | 31 +++ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/pym/portage/util/_async/PopenProcess.py b/pym/portage/util/_async/PopenProcess.py index 4344b1c9d..3fb60d527 100644 --- a/pym/portage/util/_async/PopenProcess.py +++ b/pym/portage/util/_async/PopenProcess.py @@ -1,4 +1,4 @@ -# Copyright 2012-2017 Gentoo Foundation +# Copyright 2012-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.SubProcess import SubProcess @@ -13,8 +13,7 @@ class PopenProcess(SubProcess): self._registered = True if self.pipe_reader is None: - self._reg_id = self.scheduler.child_watch_add( - self.pid, self._child_watch_cb) + self._async_waitpid() else: try: self.pipe_reader.scheduler = self.scheduler @@ -24,17 +23,25 @@ class PopenProcess(SubProcess): self.pipe_reader.start() def _pipe_reader_exit(self, pipe_reader): - self._reg_id = self.scheduler.child_watch_add( - self.pid, self._child_watch_cb) + self._async_waitpid() - def _child_watch_cb(self, pid, condition, user_data=None): - self._reg_id = None - self._waitpid_cb(pid, condition) - self.wait() + def _async_waitpid(self): + if self.returncode is None: + self.scheduler._asyncio_child_watcher.\ + add_child_handler(self.pid, self._async_waitpid_cb) + else: + self._unregister() + self._async_wait() - def _set_returncode(self, wait_retval): - SubProcess._set_returncode(self, wait_retval) + def _async_waitpid_cb(self, pid, returncode): if self.proc.returncode is None: # Suppress warning messages like this: # ResourceWarning: subprocess 1234 is still running - self.proc.returncode = self.returncode + self.proc.returncode = returncode + self._unregister() + self.returncode = returncode + self._async_wait() + + def _poll(self): + # Simply rely on _async_waitpid_cb to set the returncode. + return self.returncode
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, pym/_emerge/, pym/portage/_emirrordist/
commit: f01f504fc8d4ea7442379482889b1cee4be0fcf4 Author: Zac Medico gentoo org> AuthorDate: Sun Apr 29 01:28:42 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 29 04:31:45 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f01f504f AsynchronousTask: remove unused _wait() methods (bug 653856) Remove _wait() and _waitpid_loop() methods which are unused since AsynchronousTask.wait() now uses run_until_complete(self.async_wait()) instead. Also remove the iteration method from SchedulerInterface, since there are no more consumers. Bug: https://bugs.gentoo.org/653856 pym/_emerge/AbstractEbuildProcess.py| 27 pym/_emerge/AbstractPollTask.py | 8 +--- pym/_emerge/AsynchronousLock.py | 9 pym/_emerge/AsynchronousTask.py | 6 --- pym/_emerge/CompositeTask.py| 48 + pym/_emerge/FifoIpcDaemon.py| 13 +- pym/_emerge/PipeReader.py | 7 pym/_emerge/SpawnProcess.py | 12 +- pym/_emerge/SubProcess.py | 56 + pym/portage/_emirrordist/MirrorDistTask.py | 7 pym/portage/util/_async/AsyncScheduler.py | 25 +-- pym/portage/util/_async/AsyncTaskFuture.py | 8 pym/portage/util/_async/PipeLogger.py | 7 pym/portage/util/_async/PipeReaderBlockingIO.py | 10 - pym/portage/util/_async/SchedulerInterface.py | 1 - 15 files changed, 6 insertions(+), 238 deletions(-) diff --git a/pym/_emerge/AbstractEbuildProcess.py b/pym/_emerge/AbstractEbuildProcess.py index b10aa4bfa..03c834912 100644 --- a/pym/_emerge/AbstractEbuildProcess.py +++ b/pym/_emerge/AbstractEbuildProcess.py @@ -386,33 +386,6 @@ class AbstractEbuildProcess(SpawnProcess): if not self.cancelled: self._unexpected_exit() - def _wait(self): - """ - Override _wait to unlock self._build_dir if necessary. Normally, it - should already be unlocked, so this functions only as a failsafe. - Execution of the failsafe code will automatically become a fatal - error at the same time as event loop recursion is disabled. - """ - # SpawnProcess._wait() requires the pid, so wait here for the - # pid to become available. - while self._start_future is not None: - self.scheduler.run_until_complete(self._start_future) - - SpawnProcess._wait(self) - - if self._build_dir is not None: - self._build_dir_unlock = self._build_dir.async_unlock() - # Unlock only once. - self._build_dir = None - - if not (self._build_dir_unlock is None or - self._build_dir_unlock.done()): - # This will automatically become a fatal error at the same - # time as event loop recursion is disabled. - self.scheduler.run_until_complete(self._build_dir_unlock) - - return self.returncode - def _async_wait(self): """ Override _async_wait to asynchronously unlock self._build_dir diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py index 0aac97be5..0ce3594b4 100644 --- a/pym/_emerge/AbstractPollTask.py +++ b/pym/_emerge/AbstractPollTask.py @@ -1,4 +1,4 @@ -# Copyright 1999-2015 Gentoo Foundation +# Copyright 1999-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import array @@ -136,12 +136,6 @@ class AbstractPollTask(AsynchronousTask): self.returncode = self.returncode or os.EX_OK self._async_wait() - def _wait(self): - if self.returncode is not None: - return self.returncode - self._wait_loop() - return self.returncode - def _wait_loop(self, timeout=None): loop = getattr(self.scheduler, '_asyncio_wrapper', self.scheduler) tasks = [self.async_wait()] diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py index 2019adaab..c5991bcff 100644 --- a/pym/_emerge/AsynchronousLock.py +++ b/pym/_emerge/AsynchronousLock.py @@ -82,15 +82,6 @@ class AsynchronousLock(AsynchronousTask): self._imp.poll() return self.returncode - def _wait(self): - """ - Deprecated. Use _async_wait() instead. - """ - if self.returncode is not None: - return self.returncode - self.returncode = self._imp.wait() -
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 2ab6eed08c9eafa29de6100b19c9c072b17b9167
Author: Zac Medico gentoo org>
AuthorDate: Sun Apr 29 04:06:25 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Apr 29 04:06:25 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2ab6eed0
PipeReaderBlockingIO._reader_thread: asyncio compat (bug 653856)
Use call_soon_threadsafe for asyncio compatibility, and also
remove idle_add from SchedulerInterface since there are no more
consumers.
Bug: https://bugs.gentoo.org/653856
pym/portage/util/_async/PipeReaderBlockingIO.py | 3 +--
pym/portage/util/_async/SchedulerInterface.py | 3 +--
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/_async/PipeReaderBlockingIO.py
b/pym/portage/util/_async/PipeReaderBlockingIO.py
index b426f7695..0727d2c6a 100644
--- a/pym/portage/util/_async/PipeReaderBlockingIO.py
+++ b/pym/portage/util/_async/PipeReaderBlockingIO.py
@@ -55,7 +55,7 @@ class PipeReaderBlockingIO(AbstractPollTask):
del self._threads[f]
if not self._threads:
# Thread-safe callback to
EventLoop
-
self.scheduler.idle_add(self._eof)
+
self.scheduler.call_soon_threadsafe(self._eof)
break
f.close()
@@ -64,7 +64,6 @@ class PipeReaderBlockingIO(AbstractPollTask):
if self.returncode is None:
self.returncode = os.EX_OK
self._async_wait()
- return False
def _cancel(self):
self._terminate.set()
diff --git a/pym/portage/util/_async/SchedulerInterface.py
b/pym/portage/util/_async/SchedulerInterface.py
index ab2a70852..acacf7c31 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2013 Gentoo Foundation
+# Copyright 2012-2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import gzip
@@ -14,7 +14,6 @@ class SchedulerInterface(SlotObject):
_event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN",
"IO_NVAL", "IO_OUT", "IO_PRI",
"child_watch_add",
- "idle_add",
"io_add_watch",
"iteration",
"source_remove",
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 532366feaf8e3cb04e0d19d90498cfe0b9361ab1 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 28 23:27:49 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 28 23:27:49 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=532366fe PipeReaderBlockingIO: fix deprecated _wait usage (bug 653856) Use _async_wait() instead of wait(). Bug: https://bugs.gentoo.org/653856 pym/portage/util/_async/PipeReaderBlockingIO.py | 9 ++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pym/portage/util/_async/PipeReaderBlockingIO.py b/pym/portage/util/_async/PipeReaderBlockingIO.py index b06adf6ed..b426f7695 100644 --- a/pym/portage/util/_async/PipeReaderBlockingIO.py +++ b/pym/portage/util/_async/PipeReaderBlockingIO.py @@ -1,4 +1,4 @@ -# Copyright 2012 Gentoo Foundation +# Copyright 2012-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 try: @@ -63,7 +63,7 @@ class PipeReaderBlockingIO(AbstractPollTask): self._registered = False if self.returncode is None: self.returncode = os.EX_OK - self.wait() + self._async_wait() return False def _cancel(self): @@ -71,9 +71,12 @@ class PipeReaderBlockingIO(AbstractPollTask): self._registered = False if self.returncode is None: self.returncode = self._cancelled_returncode - self.wait() + self._async_wait() def _wait(self): + """ + Deprecated. Use _async_wait() instead. + """ if self.returncode is not None: return self.returncode self._wait_loop()
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 98e5852117cbda19b4f4266eb71a053dced64a88 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 28 23:17:30 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 28 23:17:30 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=98e58521 AsyncTaskFuture: fix deprecated _wait usage (bug 653856) Fix _done_callback to use _async_wait() instead of wait(). Bug: https://bugs.gentoo.org/653856 pym/portage/util/_async/AsyncTaskFuture.py | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_async/AsyncTaskFuture.py b/pym/portage/util/_async/AsyncTaskFuture.py index ce881ba8d..cdc842b05 100644 --- a/pym/portage/util/_async/AsyncTaskFuture.py +++ b/pym/portage/util/_async/AsyncTaskFuture.py @@ -28,9 +28,12 @@ class AsyncTaskFuture(AsynchronousTask): self.returncode = os.EX_OK else: self.returncode = 1 - self.wait() + self._async_wait() def _wait(self): + """ + Deprecated. Use _async_wait() instead. + """ if self.returncode is None: self.scheduler.run_until_complete(self.future) return self.returncode
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 3d81af6440cfd20cd007568779d6129083f1de24 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 28 22:45:56 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 28 22:45:56 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d81af64 AsyncScheduler: fix deprecated _wait usage (bug 653856) Override AsynchronousTask._async_wait() for cleanup, since AsynchronousTask._wait() is deprecated. Bug: https://bugs.gentoo.org/653856 pym/portage/util/_async/AsyncScheduler.py | 13 +++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py index 1b870c771..2ddbaa885 100644 --- a/pym/portage/util/_async/AsyncScheduler.py +++ b/pym/portage/util/_async/AsyncScheduler.py @@ -29,12 +29,11 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): def _poll(self): if not (self._is_work_scheduled() or self._keep_scheduling()): - self._cleanup() - if self._error_count > 0: self.returncode = 1 else: self.returncode = os.EX_OK + self._async_wait() return self.returncode def _cancel(self): @@ -91,7 +90,17 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): self._event_loop.source_remove(self._loadavg_check_id) self._loadavg_check_id = None + def _async_wait(self): + """ + Override _async_wait to call self._cleanup(). + """ + self._cleanup() + super(AsyncScheduler, self)._async_wait() + def _wait(self): + """ + Deprecated. Use _async_wait() instead. + """ # Loop while there are jobs to be scheduled. while self._keep_scheduling(): self._event_loop.iteration()
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/
commit: be61882996099322bb3a1e82e71f475b4141ad40 Author: Zac Medico gentoo org> AuthorDate: Tue Apr 24 23:28:08 2018 + Commit: Zac Medico gentoo org> CommitDate: Fri Apr 27 21:33:00 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=be618829 Add iter_gather function (bug 653946) This is similar to asyncio.gather, but takes an iterator of futures as input, and includes support for max_jobs and max_load parameters. For bug 653946, this will be used to asynchronously gather the results of the portdbapi.async_fetch_map calls that are required to generate a Manifest, while using the max_jobs parameter to limit the number of concurrent async_aux_get calls. Bug: https://bugs.gentoo.org/653946 pym/portage/util/futures/iter_completed.py | 73 ++ 1 file changed, 73 insertions(+) diff --git a/pym/portage/util/futures/iter_completed.py b/pym/portage/util/futures/iter_completed.py index 5ad075305..1d6a9a4bd 100644 --- a/pym/portage/util/futures/iter_completed.py +++ b/pym/portage/util/futures/iter_completed.py @@ -112,3 +112,76 @@ def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None): # cleanup in case of interruption by SIGINT, etc scheduler.cancel() scheduler.wait() + + +def iter_gather(futures, max_jobs=None, max_load=None, loop=None): + """ + This is similar to asyncio.gather, but takes an iterator of + futures as input, and includes support for max_jobs and max_load + parameters. + + @param futures: iterator of asyncio.Future (or compatible) + @type futures: iterator + @param max_jobs: max number of futures to process concurrently (default + is multiprocessing.cpu_count()) + @type max_jobs: int + @param max_load: max load allowed when scheduling a new future, + otherwise schedule no more than 1 future at a time (default + is multiprocessing.cpu_count()) + @type max_load: int or float + @param loop: event loop + @type loop: EventLoop + @return: a Future resulting in a list of done input futures, in the + same order that they were yielded from the input iterator + @rtype: asyncio.Future (or compatible) + """ + loop = loop or global_event_loop() + loop = getattr(loop, '_asyncio_wrapper', loop) + result = loop.create_future() + futures_list = [] + + def future_generator(): + for future in futures: + futures_list.append(future) + yield future + + completed_iter = async_iter_completed( + future_generator(), + max_jobs=max_jobs, + max_load=max_load, + loop=loop, + ) + + def handle_result(future_done_set): + if result.cancelled(): + if not future_done_set.cancelled(): + # All exceptions must be consumed from future_done_set, in order + # to avoid triggering the event loop's exception handler. + list(future.exception() for future in future_done_set.result() + if not future.cancelled()) + return + + try: + handle_result.current_task = next(completed_iter) + except StopIteration: + result.set_result(futures_list) + else: + handle_result.current_task.add_done_callback(handle_result) + + try: + handle_result.current_task = next(completed_iter) + except StopIteration: + handle_result.current_task = None + result.set_result(futures_list) + else: + handle_result.current_task.add_done_callback(handle_result) + + def cancel_callback(result): + if (result.cancelled() and + handle_result.current_task is not None and + not handle_result.current_task.done()): + handle_result.current_task.cancel() + + result.add_done_callback(cancel_callback) + + return result
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: a65d93bdeb4d964603cd0ce9b0a75a571c9bdefa
Author: Zac Medico gentoo org>
AuthorDate: Thu Apr 26 09:19:22 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Apr 26 09:31:09 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a65d93bd
SchedulerInterface: expose AbstractEventLoop methods (bug 591760)
Bug: https://bugs.gentoo.org/591760
pym/portage/util/_async/SchedulerInterface.py | 34 +++
1 file changed, 30 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/_async/SchedulerInterface.py
b/pym/portage/util/_async/SchedulerInterface.py
index 21420ae41..ab2a70852 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -13,10 +13,36 @@ class SchedulerInterface(SlotObject):
_event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN",
"IO_NVAL", "IO_OUT", "IO_PRI",
- "call_soon", "call_soon_threadsafe",
- "child_watch_add", "create_future",
- "idle_add", "io_add_watch", "iteration", "run_until_complete",
- "source_remove", "timeout_add")
+ "child_watch_add",
+ "idle_add",
+ "io_add_watch",
+ "iteration",
+ "source_remove",
+ "timeout_add",
+
+ "add_reader",
+ "add_writer",
+ "call_at",
+ "call_exception_handler",
+ "call_later",
+ "call_soon",
+ "call_soon_threadsafe",
+ "close",
+ "create_future",
+ "default_exception_handler",
+ "get_debug",
+ "is_closed",
+ "is_running",
+ "remove_reader",
+ "remove_writer",
+ "run_in_executor",
+ "run_until_complete",
+ "set_debug",
+ "time",
+
+ "_asyncio_child_watcher",
+ "_asyncio_wrapper",
+ )
__slots__ = _event_loop_attrs + ("_event_loop", "_is_background")
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: d16206ccd315edbd4bd7a46c0b1aa9e59a46db43 Author: Zac Medico gentoo org> AuthorDate: Thu Apr 26 07:45:32 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 26 08:02:28 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d16206cc AsyncScheduler._poll(): fix event loop recursion (bug 591760) Call self._cleanup() and set the returncode, in order to avoid event loop recursion in self.wait(). Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/AsyncScheduler.py | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py index b89b57dab..1b870c771 100644 --- a/pym/portage/util/_async/AsyncScheduler.py +++ b/pym/portage/util/_async/AsyncScheduler.py @@ -29,7 +29,12 @@ class AsyncScheduler(AsynchronousTask, PollScheduler): def _poll(self): if not (self._is_work_scheduled() or self._keep_scheduling()): - self.wait() + self._cleanup() + + if self._error_count > 0: + self.returncode = 1 + else: + self.returncode = os.EX_OK return self.returncode def _cancel(self):
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 384b61dd24a21aaccd7c643e58b07ea56c3cbdfc Author: Zac Medico gentoo org> AuthorDate: Thu Apr 26 05:31:54 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 26 05:46:30 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=384b61dd PipeLogger._output_handler: use _async_wait() (bug 591760) Use _async_wait() to avoid event loop recursion. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/PipeLogger.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/_async/PipeLogger.py b/pym/portage/util/_async/PipeLogger.py index aa605d94d..ed1202edf 100644 --- a/pym/portage/util/_async/PipeLogger.py +++ b/pym/portage/util/_async/PipeLogger.py @@ -1,4 +1,4 @@ -# Copyright 2008-2013 Gentoo Foundation +# Copyright 2008-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import fcntl @@ -89,7 +89,8 @@ class PipeLogger(AbstractPollTask): if not buf: # EOF self._unregister() - self.wait() + self.returncode = self.returncode or os.EX_OK + self._async_wait() break else:
[gentoo-commits] proj/portage:master commit in: pym/portage/util/, pym/portage/util/_async/, pym/_emerge/
commit: 6c865d6b3130de63dc3d534d2a0596dd59d5e662
Author: Zac Medico gentoo org>
AuthorDate: Tue Apr 24 02:21:59 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Apr 26 03:19:21 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c865d6b
AsynchronousTask: add scheduler attribute (bug 653856)
Add an AsynchronousTask.scheduler attribute, which will hold an event
loop instance in order to work with Future instances. Name the
attribute "scheduler" because this is the name used by many existing
subclasses of AsynchronousTask. Since the AsyncScheduler class already
has an _event_loop attribute that it inherits from PollScheduler, use
a property for compatibility. Also update the SlotObject constructor
to allow a subclass to override an attribute from __slots__ with a
property, so that AsyncScheduler can use a property for the scheduler
attribute.
Bug: https://bugs.gentoo.org/653856
pym/_emerge/AbstractPollTask.py| 3 +--
pym/_emerge/AsynchronousLock.py| 2 +-
pym/_emerge/AsynchronousTask.py| 2 +-
pym/_emerge/CompositeTask.py | 2 +-
pym/portage/util/SlotObject.py | 9 -
pym/portage/util/_async/AsyncScheduler.py | 7 +++
pym/portage/util/_async/AsyncTaskFuture.py | 2 +-
7 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py
index 0d38bd481..aa8fc6593 100644
--- a/pym/_emerge/AbstractPollTask.py
+++ b/pym/_emerge/AbstractPollTask.py
@@ -11,8 +11,7 @@ from _emerge.AsynchronousTask import AsynchronousTask
class AbstractPollTask(AsynchronousTask):
- __slots__ = ("scheduler",) + \
- ("_registered",)
+ __slots__ = ("_registered",)
_bufsize = 4096
diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py
index 78ab37ecb..0178feab2 100644
--- a/pym/_emerge/AsynchronousLock.py
+++ b/pym/_emerge/AsynchronousLock.py
@@ -37,7 +37,7 @@ class AsynchronousLock(AsynchronousTask):
signals to the main thread).
"""
- __slots__ = ('path', 'scheduler',) + \
+ __slots__ = ('path',) + \
('_imp', '_force_async', '_force_dummy', '_force_process', \
'_force_thread', '_unlock_future')
diff --git a/pym/_emerge/AsynchronousTask.py b/pym/_emerge/AsynchronousTask.py
index da58261db..e29324440 100644
--- a/pym/_emerge/AsynchronousTask.py
+++ b/pym/_emerge/AsynchronousTask.py
@@ -16,7 +16,7 @@ class AsynchronousTask(SlotObject):
the task is complete and self.returncode has been set.
"""
- __slots__ = ("background", "cancelled", "returncode") + \
+ __slots__ = ("background", "cancelled", "returncode", "scheduler") + \
("_exit_listeners", "_exit_listener_stack", "_start_listeners",
"_waiting")
diff --git a/pym/_emerge/CompositeTask.py b/pym/_emerge/CompositeTask.py
index f3acc9696..bfd4bacbd 100644
--- a/pym/_emerge/CompositeTask.py
+++ b/pym/_emerge/CompositeTask.py
@@ -6,7 +6,7 @@ from portage import os
class CompositeTask(AsynchronousTask):
- __slots__ = ("scheduler",) + ("_current_task",)
+ __slots__ = ("_current_task",)
_TASK_QUEUED = -1
diff --git a/pym/portage/util/SlotObject.py b/pym/portage/util/SlotObject.py
index 4bb682258..ba6215874 100644
--- a/pym/portage/util/SlotObject.py
+++ b/pym/portage/util/SlotObject.py
@@ -20,7 +20,14 @@ class SlotObject(object):
raise AssertionError(
"class '%s' duplicates '%s'
value in __slots__ of base class '%s'" %
(self.__class__.__name__,
myattr, c.__name__))
- setattr(self, myattr, myvalue)
+ try:
+ setattr(self, myattr, myvalue)
+ except AttributeError:
+ # Allow a property to override a
__slots__ value, but raise an
+ # error if the intended value is
something other than None.
+ if not (myvalue is None and
+ isinstance(getattr(type(self),
myattr, None), property)):
+ raise
if kwargs:
raise TypeError(
diff --git a/pym/portage/util/_async/AsyncScheduler.py
b/pym/portage/util/_async/AsyncScheduler.py
index 9beb8a848..b89b57dab 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -20,6 +20,13 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
self._remaining_tasks = True
self._loadavg_check_id = None
+ @property
+ def scheduler(self):
+ """
+
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 3eb46b05eb098507ea74108559bf77006e0f8142 Author: Zac Medico gentoo org> AuthorDate: Mon Apr 23 00:49:15 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 23 00:49:15 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3eb46b05 FileDigester: use _async_waitpid (bug 591760) When pid exit status is not yet available, use the Subprocess _async_waitpid() method to avoid event loop recursion. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/FileDigester.py | 7 +-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/_async/FileDigester.py b/pym/portage/util/_async/FileDigester.py index 881c69280..72f06759c 100644 --- a/pym/portage/util/_async/FileDigester.py +++ b/pym/portage/util/_async/FileDigester.py @@ -60,8 +60,11 @@ class FileDigester(ForkProcess): def _digest_pipe_reader_exit(self, pipe_reader): self._parse_digests(pipe_reader.getvalue()) self._digest_pipe_reader = None - self._unregister() - self.wait() + if self.pid is None: + self._unregister() + self._async_wait() + else: + self._async_waitpid() def _unregister(self): ForkProcess._unregister(self)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: d836b0d32797f1431ebeaa20e2a18ea5cb7c3f7c Author: Zac Medico gentoo org> AuthorDate: Sun Apr 22 15:56:29 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 22 17:48:52 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d836b0d3 AsyncFunction: use _async_waitpid (bug 591760) When pid exit status is not yet available, use the Subprocess _async_waitpid() method to avoid event loop recursion. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_async/AsyncFunction.py | 7 +-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/_async/AsyncFunction.py b/pym/portage/util/_async/AsyncFunction.py index 40f6c5e75..ad3d8333f 100644 --- a/pym/portage/util/_async/AsyncFunction.py +++ b/pym/portage/util/_async/AsyncFunction.py @@ -57,8 +57,11 @@ class AsyncFunction(ForkProcess): # and returned an unsuccessful returncode. pass self._async_func_reader = None - self._unregister() - self.wait() + if self.returncode is None: + self._async_waitpid() + else: + self._unregister() + self._async_wait() def _unregister(self): ForkProcess._unregister(self)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, pym/portage/tests/util/futures/
commit: 6529d0e0ec017c4fa3048235649d25e71c967941
Author: Zac Medico gentoo org>
AuthorDate: Sun Apr 22 17:27:32 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Apr 22 17:27:32 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6529d0e0
test_iter_completed: fix max_load in testAsyncCancel
Since async_iter_completed sets a default multiprocessing.cpu_count()
value when max_load is None, use max_load=True to disable the load
limit.
Fixes: a9e8ebaa6979 ("Add async_iter_completed for asyncio migration (bug
591760)")
pym/portage/tests/util/futures/test_iter_completed.py | 2 +-
pym/portage/util/_async/AsyncScheduler.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/portage/tests/util/futures/test_iter_completed.py
b/pym/portage/tests/util/futures/test_iter_completed.py
index b07146ed3..71343c22d 100644
--- a/pym/portage/tests/util/futures/test_iter_completed.py
+++ b/pym/portage/tests/util/futures/test_iter_completed.py
@@ -69,7 +69,7 @@ class IterCompletedTestCase(TestCase):
yield future
for future_done_set in async_iter_completed(future_generator(),
- max_jobs=True, max_load=None, loop=loop):
+ max_jobs=True, max_load=True, loop=loop):
future_done_set.cancel()
break
diff --git a/pym/portage/util/_async/AsyncScheduler.py
b/pym/portage/util/_async/AsyncScheduler.py
index 3deb6cb04..9beb8a848 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -14,7 +14,7 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
if max_jobs is None:
max_jobs = 1
self._max_jobs = max_jobs
- self._max_load = max_load
+ self._max_load = None if max_load is True else max_load
self._error_count = 0
self._running_tasks = set()
self._remaining_tasks = True
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: da5efb5e677d12c850ff02140e24a095a8efdafb
Author: Zac Medico gentoo org>
AuthorDate: Thu Apr 19 06:36:32 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Apr 19 06:36:32 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=da5efb5e
EventLoop.run_until_complete: remove unneeded partial
Fixes: 25245d7eb86e ("EventLoop.run_until_complete: wait for done callbacks")
pym/portage/util/_eventloop/EventLoop.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 33fae26f2..6a8b906ed 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -818,7 +818,7 @@ class EventLoop(object):
# is easily achieved by registering a done callback and waiting
for
# it to execute.
waiter = self.create_future()
- future.add_done_callback(functools.partial(waiter.set_result))
+ future.add_done_callback(waiter.set_result)
while not waiter.done():
self.iteration()
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 128c13ff8bd9a1685fb844f7a83469b2271f5e0f Author: Zac Medico gentoo org> AuthorDate: Thu Apr 19 06:19:00 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 19 06:25:32 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=128c13ff AsyncTaskFuture: override _wait method If the _wait method is called before the future is done, then wait for it to complete. pym/portage/util/_async/AsyncTaskFuture.py | 5 + 1 file changed, 5 insertions(+) diff --git a/pym/portage/util/_async/AsyncTaskFuture.py b/pym/portage/util/_async/AsyncTaskFuture.py index ee39183fe..c2316deeb 100644 --- a/pym/portage/util/_async/AsyncTaskFuture.py +++ b/pym/portage/util/_async/AsyncTaskFuture.py @@ -29,3 +29,8 @@ class AsyncTaskFuture(AsynchronousTask): else: self.returncode = 1 self.wait() + + def _wait(self): + if self.returncode is None: + self.scheduler.run_until_complete(self.future) + return self.returncode
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 25245d7eb86ed197b7d7cfead0dbe4ce8ad4bc5b Author: Zac Medico gentoo org> AuthorDate: Thu Apr 19 06:13:33 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 19 06:13:33 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=25245d7e EventLoop.run_until_complete: wait for done callbacks Since done callbacks are executed via call_soon, it's desirable to continue iterating until those callbacks have executed, which is easily achieved by registering a done callback and waiting for it to execute. pym/portage/util/_eventloop/EventLoop.py | 9 - 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 7208c3aa1..33fae26f2 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -812,7 +812,14 @@ class EventLoop(object): @raise: the Future's exception """ future = asyncio.ensure_future(future, loop=self._asyncio_wrapper) - while not future.done(): + + # Since done callbacks are executed via call_soon, it's desirable + # to continue iterating until those callbacks have executed, which + # is easily achieved by registering a done callback and waiting for + # it to execute. + waiter = self.create_future() + future.add_done_callback(functools.partial(waiter.set_result)) + while not waiter.done(): self.iteration() return future.result()
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: b443b87c5397867c287f3bc4c5f1f4fa5e234d0a
Author: Zac Medico gentoo org>
AuthorDate: Wed Apr 18 21:38:33 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Apr 18 21:49:42 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b443b87c
EventLoop._run_idle_callbacks: support recursive calls (bug 653508)
Since recursive calls must be supported until all consumers of the
AsynchronousLock.unlock() method have been migrated to use the
async_unlock() method (bug 614108.), support recursive calls by using
a self._idle_callbacks_remaining loop control variable that is reset
by each recursive call.
Fixes: 9772f8f2a58a (EventLoop._idle_add: use thread-safe deque append)
Bug: https://bugs.gentoo.org/653508
pym/portage/util/_eventloop/EventLoop.py | 60 ++--
1 file changed, 26 insertions(+), 34 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 895303699..7208c3aa1 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -145,7 +145,7 @@ class EventLoop(object):
# Use deque, with thread-safe append, in order to emulate the
FIFO
# queue behavior of the AbstractEventLoop.call_soon method.
self._idle_callbacks = collections.deque()
- self._idle_callbacks_running = False
+ self._idle_callbacks_remaining = 0
self._timeout_handlers = {}
self._timeout_interval = None
self._default_executor = None
@@ -534,26 +534,29 @@ class EventLoop(object):
reschedule = []
# Use remaining count to avoid calling any newly scheduled
callbacks,
# since self._idle_callbacks can be modified during the
exection of
- # these callbacks.
- remaining = len(self._idle_callbacks)
- try:
- while remaining:
- remaining -= 1
- try:
- x = self._idle_callbacks.popleft() #
thread-safe
- except IndexError:
- break
- if x._cancelled:
- # it got cancelled while executing
another callback
- continue
- if x._callback(*x._args):
- reschedule.append(x)
- else:
- x._cancelled = True
- state_change += 1
- finally:
- # Reschedule those that were not cancelled.
- self._idle_callbacks.extend(reschedule)
+ # these callbacks. The remaining count can be reset by recursive
+ # calls to this method. Recursion must remain supported until
all
+ # consumers of AsynchronousLock.unlock() have been migrated to
the
+ # async_unlock() method, see bug 614108.
+ self._idle_callbacks_remaining = len(self._idle_callbacks)
+
+ while self._idle_callbacks_remaining:
+ self._idle_callbacks_remaining -= 1
+ try:
+ x = self._idle_callbacks.popleft() # thread-safe
+ except IndexError:
+ break
+ if x._cancelled:
+ # it got cancelled while executing another
callback
+ continue
+ if x._callback(*x._args):
+ # Reschedule, but not until after it's called,
since
+ # we don't want it to call itself in a
recursive call
+ # to this method.
+ self._idle_callbacks.append(x)
+ else:
+ x._cancelled = True
+ state_change += 1
return bool(state_change)
@@ -587,19 +590,8 @@ class EventLoop(object):
with self._thread_rlock:
- if self._idle_callbacks_running:
- # The caller should use call_soon in order to
- # prevent recursion here. Raise an error because
- # _run_idle_callbacks has an internal remaining
- # count that recursion would render meaningless.
- raise AssertionError('idle callback recursion')
-
- self._idle_callbacks_running = True
- try:
- if self._run_idle_callbacks():
-
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: d36f8b2c9c43311f4c1333afe7ce1cc7a147b836
Author: Zac Medico gentoo org>
AuthorDate: Tue Apr 17 18:19:02 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Apr 17 18:19:02 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d36f8b2c
EventLoop._run_idle_callbacks: make recursive calls fatal
The caller should use call_soon in order to prevent recursion
here. Raise an error because _run_idle_callbacks has an internal
remaining count that recursion would render meaningless.
Fixes: 9772f8f2a58a (EventLoop._idle_add: use thread-safe deque append)
pym/portage/util/_eventloop/EventLoop.py | 16 ++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index a61a3d74a..895303699 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -145,6 +145,7 @@ class EventLoop(object):
# Use deque, with thread-safe append, in order to emulate the
FIFO
# queue behavior of the AbstractEventLoop.call_soon method.
self._idle_callbacks = collections.deque()
+ self._idle_callbacks_running = False
self._timeout_handlers = {}
self._timeout_interval = None
self._default_executor = None
@@ -586,8 +587,19 @@ class EventLoop(object):
with self._thread_rlock:
- if self._run_idle_callbacks():
- calls += 1
+ if self._idle_callbacks_running:
+ # The caller should use call_soon in order to
+ # prevent recursion here. Raise an error because
+ # _run_idle_callbacks has an internal remaining
+ # count that recursion would render meaningless.
+ raise AssertionError('idle callback recursion')
+
+ self._idle_callbacks_running = True
+ try:
+ if self._run_idle_callbacks():
+ calls += 1
+ finally:
+ self._idle_callbacks_running = False
if not self._timeout_handlers:
return bool(calls)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 0f7c9a73a805af5ec70da587b3c7d7f59dabe5ce
Author: Zac Medico gentoo org>
AuthorDate: Tue Apr 17 17:57:04 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Apr 17 17:59:10 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0f7c9a73
EventLoop._run_idle_callbacks: remove recursion check
This recursion check does not really work because callbacks are
removed from self._idle_callbacks before recursion would occur.
Fixes: 9772f8f2a58a (EventLoop._idle_add: use thread-safe deque append)
pym/portage/util/_eventloop/EventLoop.py | 22 ++
1 file changed, 6 insertions(+), 16 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index c38a4defd..a61a3d74a 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -55,7 +55,7 @@ class EventLoop(object):
__slots__ = ("callback", "data", "pid", "source_id")
class _idle_callback_class(SlotObject):
- __slots__ = ("_args", "_callback", "_calling", "_cancelled")
+ __slots__ = ("_args", "_callback", "_cancelled")
class _io_handler_class(SlotObject):
__slots__ = ("args", "callback", "f", "source_id")
@@ -545,21 +545,11 @@ class EventLoop(object):
if x._cancelled:
# it got cancelled while executing
another callback
continue
- if x._calling:
- # The caller should use call_soon in
order to prevent
- # recursion here. Raise an error
because recursive
- # calls would make the remaining count
for this loop
- # meaningless.
- raise AssertionError('recursive idle
callback')
- x._calling = True
- try:
- if x._callback(*x._args):
- reschedule.append(x)
- else:
- x._cancelled = True
- state_change += 1
- finally:
- x._calling = False
+ if x._callback(*x._args):
+ reschedule.append(x)
+ else:
+ x._cancelled = True
+ state_change += 1
finally:
# Reschedule those that were not cancelled.
self._idle_callbacks.extend(reschedule)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/, pym/portage/tests/util/futures/
commit: a9e8ebaa6979ccf0bb385e457d695bedc7b65bf5 Author: Zac Medico gentoo org> AuthorDate: Tue Apr 17 09:51:36 2018 + Commit: Zac Medico gentoo org> CommitDate: Tue Apr 17 17:44:58 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a9e8ebaa Add async_iter_completed for asyncio migration (bug 591760) This serves as a wrapper around portage's internal TaskScheduler class, allowing TaskScheduler API consumers to be migrated to use asyncio interfaces. Bug: https://bugs.gentoo.org/591760 .../tests/util/futures/test_iter_completed.py | 37 - pym/portage/util/futures/iter_completed.py | 61 +++--- 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/pym/portage/tests/util/futures/test_iter_completed.py b/pym/portage/tests/util/futures/test_iter_completed.py index 9c23aefb1..1344523c6 100644 --- a/pym/portage/tests/util/futures/test_iter_completed.py +++ b/pym/portage/tests/util/futures/test_iter_completed.py @@ -5,7 +5,11 @@ import time from portage.tests import TestCase from portage.util._async.ForkProcess import ForkProcess from portage.util._eventloop.global_event_loop import global_event_loop -from portage.util.futures.iter_completed import iter_completed +from portage.util.futures import asyncio +from portage.util.futures.iter_completed import ( + iter_completed, + async_iter_completed, +) class SleepProcess(ForkProcess): @@ -48,3 +52,34 @@ class IterCompletedTestCase(TestCase): for seconds, future in zip(expected_order, iter_completed(future_generator(), max_jobs=True, max_load=None, loop=loop)): self.assertEqual(seconds, future.result()) + + def testAsyncCancel(self): + + loop = global_event_loop()._asyncio_wrapper + input_futures = set() + future_count = 3 + + def future_generator(): + for i in range(future_count): + future = loop.create_future() + loop.call_soon(lambda future: None if future.done() + else future.set_result(None), future) + input_futures.add(future) + yield future + + for future_done_set in async_iter_completed(future_generator(), + max_jobs=True, max_load=None, loop=loop): + future_done_set.cancel() + break + + # With max_jobs=True, async_iter_completed should have executed + # the generator until it raised StopIteration. + self.assertEqual(future_count, len(input_futures)) + + loop.run_until_complete(asyncio.wait(input_futures, loop=loop)) + + # The futures may have results or they may have been cancelled + # by TaskScheduler, and behavior varies depending on the python + # interpreter. + for future in input_futures: + future.cancelled() or future.result() diff --git a/pym/portage/util/futures/iter_completed.py b/pym/portage/util/futures/iter_completed.py index 8d324de84..5ad075305 100644 --- a/pym/portage/util/futures/iter_completed.py +++ b/pym/portage/util/futures/iter_completed.py @@ -1,6 +1,7 @@ # Copyright 2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import functools import multiprocessing from portage.util._async.AsyncTaskFuture import AsyncTaskFuture @@ -31,6 +32,38 @@ def iter_completed(futures, max_jobs=None, max_load=None, loop=None): """ loop = loop or global_event_loop() loop = getattr(loop, '_asyncio_wrapper', loop) + + for future_done_set in async_iter_completed(futures, + max_jobs=max_jobs, max_load=max_load, loop=loop): + for future in loop.run_until_complete(future_done_set): + yield future + + +def async_iter_completed(futures, max_jobs=None, max_load=None, loop=None): + """ + An asynchronous version of iter_completed. This yields futures, which + when done, result in a set of input futures that are done. This serves + as a wrapper around portage's internal TaskScheduler class, using + standard asyncio interfaces. + + @param futures: iterator of asyncio.Future (or compatible) + @type futures: iterator + @param max_jobs: max number of futures to process concurrently (default + is multiprocessing.cpu_count()) + @type max_jobs: int + @param max_load: max load allowed when scheduling a new future, + otherwise schedule no more than 1 future at a time (default + is multiprocessing.cpu_count()) + @type max_load: int or float + @param loop: event loop + @type loop: EventLoop +
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 0f8e3cd3cc695e721a8b1f7cfc56c53aca19fe4d
Author: Zac Medico gentoo org>
AuthorDate: Tue Apr 17 06:20:45 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Apr 17 06:20:45 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0f8e3cd3
EventLoop._run_idle_callbacks: make recursive callbacks fatal
The caller should use call_soon in order to prevent recursion
here. Raise an error because recursive calls would make the
remaining count for this loop meaningless.
Fixes: 9772f8f2a58a (EventLoop._idle_add: use thread-safe deque append)
pym/portage/util/_eventloop/EventLoop.py | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index d4f20c6ed..c38a4defd 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -546,8 +546,11 @@ class EventLoop(object):
# it got cancelled while executing
another callback
continue
if x._calling:
- # don't call it recursively
- continue
+ # The caller should use call_soon in
order to prevent
+ # recursion here. Raise an error
because recursive
+ # calls would make the remaining count
for this loop
+ # meaningless.
+ raise AssertionError('recursive idle
callback')
x._calling = True
try:
if x._callback(*x._args):
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 9772f8f2a58a858a80ad1542d1ce46193616be67
Author: Zac Medico gentoo org>
AuthorDate: Mon Apr 16 23:55:14 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Apr 17 00:49:09 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9772f8f2
EventLoop._idle_add: use thread-safe deque append
This fixes previously unsafe usage of self._idle_callbacks when it was
a dictionary. The deque append is thread-safe, but it does *not* notify
the loop's thread, so the caller must notify if appropriate.
Fixes: 1ee8971ba1cb ("EventLoop: eliminate thread safety from call_soon")
pym/portage/util/_eventloop/EventLoop.py | 90
1 file changed, 58 insertions(+), 32 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index ae5a0a70a..d4f20c6ed 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -3,6 +3,7 @@
from __future__ import division
+import collections
import errno
import functools
import logging
@@ -30,7 +31,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures.unix_events:_PortageEventLoop,_PortageChildWatcher',
)
-from portage import OrderedDict
from portage.util import writemsg_level
from portage.util.monotonic import monotonic
from ..SlotObject import SlotObject
@@ -55,7 +55,7 @@ class EventLoop(object):
__slots__ = ("callback", "data", "pid", "source_id")
class _idle_callback_class(SlotObject):
- __slots__ = ("args", "callback", "calling", "source_id")
+ __slots__ = ("_args", "_callback", "_calling", "_cancelled")
class _io_handler_class(SlotObject):
__slots__ = ("args", "callback", "f", "source_id")
@@ -141,10 +141,10 @@ class EventLoop(object):
# If this attribute has changed since the last time that the
# call_soon callbacks have been called, then it's not safe to
# wait on self._thread_condition without a timeout.
- self._call_soon_id = 0
- # Use OrderedDict in order to emulate the FIFO queue behavior
- # of the AbstractEventLoop.call_soon method.
- self._idle_callbacks = OrderedDict()
+ self._call_soon_id = None
+ # Use deque, with thread-safe append, in order to emulate the
FIFO
+ # queue behavior of the AbstractEventLoop.call_soon method.
+ self._idle_callbacks = collections.deque()
self._timeout_handlers = {}
self._timeout_interval = None
self._default_executor = None
@@ -298,7 +298,10 @@ class EventLoop(object):
events_handled += 1
timeouts_checked = True
- call_soon = prev_call_soon_id !=
self._call_soon_id
+ call_soon = prev_call_soon_id is not
self._call_soon_id
+ if self._call_soon_id is not None and
self._call_soon_id._cancelled:
+ # Allow garbage collection of cancelled
callback.
+ self._call_soon_id = None
if (not call_soon and not event_handlers
and not events_handled and may_block):
@@ -501,8 +504,9 @@ class EventLoop(object):
@type callback: callable
@param callback: a function to call
- @rtype: int
- @return: an integer ID
+ @return: a handle which can be used to cancel the callback
+ via the source_remove method
+ @rtype: object
"""
with self._thread_condition:
source_id = self._idle_add(callback, *args)
@@ -511,32 +515,51 @@ class EventLoop(object):
def _idle_add(self, callback, *args):
"""Like idle_add(), but without thread safety."""
- source_id = self._call_soon_id = self._new_source_id()
- self._idle_callbacks[source_id] = self._idle_callback_class(
- args=args, callback=callback, source_id=source_id)
- return source_id
+ # Hold self._thread_condition when assigning self._call_soon_id,
+ # since it might be modified via a thread-safe method.
+ with self._thread_condition:
+ handle = self._call_soon_id = self._idle_callback_class(
+ _args=args, _callback=callback)
+ # This deque append is thread-safe, but it does *not* notify the
+ # loop's thread, so the caller must notify if appropriate.
+ self._idle_callbacks.append(handle)
+ return handle
def
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/, pym/portage/util/futures/_asyncio/
commit: f0db495139aa96803ec61259f3367564f6199466 Author: Zac Medico gentoo org> AuthorDate: Mon Apr 16 19:26:12 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 16 19:26:47 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f0db4951 EventLoop: refer to asyncio.Future (uses compat shim in python2) pym/portage/util/_eventloop/EventLoop.py | 3 +-- pym/portage/util/futures/_asyncio/__init__.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 4ef600a5b..ae5a0a70a 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -26,7 +26,6 @@ except ImportError: import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.futures:asyncio', - 'portage.util.futures.futures:Future', 'portage.util.futures.executor.fork:ForkExecutor', 'portage.util.futures.unix_events:_PortageEventLoop,_PortageChildWatcher', ) @@ -207,7 +206,7 @@ class EventLoop(object): """ Create a Future object attached to the loop. """ - return Future(loop=self._asyncio_wrapper) + return asyncio.Future(loop=self._asyncio_wrapper) def _new_source_id(self): """ diff --git a/pym/portage/util/futures/_asyncio/__init__.py b/pym/portage/util/futures/_asyncio/__init__.py index eca4ea284..d1584fdea 100644 --- a/pym/portage/util/futures/_asyncio/__init__.py +++ b/pym/portage/util/futures/_asyncio/__init__.py @@ -6,6 +6,7 @@ __all__ = ( 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ensure_future', + 'Future', 'get_child_watcher', 'get_event_loop', 'set_child_watcher',
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 1ee8971ba1cb34e6b3cd3d5fda23066b24630e3a Author: Zac Medico gentoo org> AuthorDate: Mon Apr 16 08:29:36 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 16 08:46:21 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1ee8971b EventLoop: eliminate thread safety from call_soon The call_soon method is used heavily by asyncio.Task to execute coroutine steps, so it's important to eliminate the overhead associated with thread safety. pym/portage/util/_eventloop/EventLoop.py | 20 ++-- 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 38e735999..4ef600a5b 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -506,12 +506,17 @@ class EventLoop(object): @return: an integer ID """ with self._thread_condition: - source_id = self._call_soon_id = self._new_source_id() - self._idle_callbacks[source_id] = self._idle_callback_class( - args=args, callback=callback, source_id=source_id) + source_id = self._idle_add(callback, *args) self._thread_condition.notify() return source_id + def _idle_add(self, callback, *args): + """Like idle_add(), but without thread safety.""" + source_id = self._call_soon_id = self._new_source_id() + self._idle_callbacks[source_id] = self._idle_callback_class( + args=args, callback=callback, source_id=source_id) + return source_id + def _run_idle_callbacks(self): # assumes caller has acquired self._thread_rlock if not self._idle_callbacks: @@ -810,11 +815,14 @@ class EventLoop(object): @return: a handle which can be used to cancel the callback @rtype: asyncio.Handle (or compatible) """ - return self._handle(self.idle_add( + return self._handle(self._idle_add( self._call_soon_callback(callback, args)), self) - # The call_soon method inherits thread safety from the idle_add method. - call_soon_threadsafe = call_soon + def call_soon_threadsafe(self, callback, *args): + """Like call_soon(), but thread safe.""" + # idle_add provides thread safety + return self._handle(self.idle_add( + self._call_soon_callback(callback, args)), self) def time(self): """Return the time according to the event loop's clock.
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: dc7faa37c3b5b9cbb9736d4b1669510db1b953d6
Author: Zac Medico gentoo org>
AuthorDate: Mon Apr 16 06:40:16 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Apr 16 06:40:16 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dc7faa37
EventLoop: fix add_reader/writer to call source_remove
The previous instance of self._selector_callback has to be removed
before replacing it with a new instance.
Fixes: 2b6e90fadfb1 ("EventLoop: support add_reader/writer fd overlap (bug
649588)")
pym/portage/util/_eventloop/EventLoop.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 0024d162c..38e735999 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -620,6 +620,7 @@ class EventLoop(object):
if mask != self._EVENT_READ:
selector_mask |= mask
callbacks.append(item)
+ self.source_remove(handler.source_id)
self.io_add_watch(fd, selector_mask,
self._selector_callback(callbacks))
def remove_reader(self, fd):
@@ -667,6 +668,7 @@ class EventLoop(object):
if mask != self._EVENT_WRITE:
selector_mask |= mask
callbacks.append(item)
+ self.source_remove(handler.source_id)
self.io_add_watch(fd, selector_mask,
self._selector_callback(callbacks))
def remove_writer(self, fd):
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/, pym/portage/util/futures/
commit: 9ebbe247f9c92bc042a67d24cadb314f4f0a5319 Author: Zac Medico gentoo org> AuthorDate: Mon Apr 16 06:13:09 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 16 06:17:52 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9ebbe247 Eventloop.run_until_complete: call asyncio.ensure_future This eliminates the need for _PortageEventLoop to override the Eventloop.run_until_complete method. pym/portage/util/_eventloop/EventLoop.py | 2 ++ pym/portage/util/futures/unix_events.py | 15 +-- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 35d0a35ba..0024d162c 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -25,6 +25,7 @@ except ImportError: import portage portage.proxy.lazyimport.lazyimport(globals(), + 'portage.util.futures:asyncio', 'portage.util.futures.futures:Future', 'portage.util.futures.executor.fork:ForkExecutor', 'portage.util.futures.unix_events:_PortageEventLoop,_PortageChildWatcher', @@ -781,6 +782,7 @@ class EventLoop(object): @return: the Future's result @raise: the Future's exception """ + future = asyncio.ensure_future(future, loop=self._asyncio_wrapper) while not future.done(): self.iteration() diff --git a/pym/portage/util/futures/unix_events.py b/pym/portage/util/futures/unix_events.py index 9d84ab6aa..d69b13718 100644 --- a/pym/portage/util/futures/unix_events.py +++ b/pym/portage/util/futures/unix_events.py @@ -30,7 +30,6 @@ from portage.util.futures import ( asyncio, events, ) -from portage.util.futures.futures import Future class _PortageEventLoop(events.AbstractEventLoop): @@ -45,6 +44,7 @@ class _PortageEventLoop(events.AbstractEventLoop): @param loop: an instance of portage's internal event loop """ self._loop = loop + self.run_until_complete = loop.run_until_complete self.call_soon = loop.call_soon self.call_soon_threadsafe = loop.call_soon_threadsafe self.call_later = loop.call_later @@ -64,19 +64,6 @@ class _PortageEventLoop(events.AbstractEventLoop): self.set_debug = loop.set_debug self.get_debug = loop.get_debug - def run_until_complete(self, future): - """ - Run the event loop until a Future is done. - - @type future: asyncio.Future - @param future: a Future to wait for - @rtype: object - @return: the Future's result - @raise: the Future's exception - """ - return self._loop.run_until_complete( - asyncio.ensure_future(future, loop=self)) - def create_task(self, coro): """ Schedule a coroutine object.
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/, pym/portage/tests/util/futures/asyncio/
commit: 9a7b0a006e65f8683716d60574e4f19f8ffd603d
Author: Zac Medico gentoo org>
AuthorDate: Sat Apr 14 21:29:29 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Apr 16 00:04:26 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9a7b0a00
Implement AbstractEventLoop.connect_read_pipe (bug 649588)
In python versions that support asyncio, this allows API consumers
to use subprocess.PIPE for asyncio.create_subprocess_exec() stdout
and stderr parameters.
Bug: https://bugs.gentoo.org/649588
.../util/futures/asyncio/test_subprocess_exec.py | 30
pym/portage/util/futures/unix_events.py| 157 -
2 files changed, 184 insertions(+), 3 deletions(-)
diff --git a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
index d30f48c43..94984fc93 100644
--- a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
+++ b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
@@ -2,6 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
import os
+import subprocess
from portage.process import find_binary
from portage.tests import TestCase
@@ -161,3 +162,32 @@ class SubprocessExecTestCase(TestCase):
f.close()
self._run_test(test)
+
+ def testReadTransport(self):
+ """
+ Test asyncio.create_subprocess_exec(stdout=subprocess.PIPE)
which
+ requires an AbstractEventLoop.connect_read_pipe implementation
+ (and a ReadTransport implementation for it to return).
+ """
+ if not hasattr(asyncio, 'create_subprocess_exec'):
+ self.skipTest('create_subprocess_exec not implemented
for python2')
+
+ args_tuple = (b'hello', b'world')
+ echo_binary = find_binary("echo")
+ self.assertNotEqual(echo_binary, None)
+ echo_binary = echo_binary.encode()
+
+ def test(loop):
+ with open(os.devnull, 'rb', 0) as devnull:
+ proc = loop.run_until_complete(
+ asyncio.create_subprocess_exec(
+ echo_binary, *args_tuple,
+ stdin=devnull,
+ stdout=subprocess.PIPE,
stderr=subprocess.STDOUT))
+
+ self.assertEqual(
+
tuple(loop.run_until_complete(proc.stdout.read()).split()),
+ args_tuple)
+ self.assertEqual(loop.run_until_complete(proc.wait()),
os.EX_OK)
+
+ self._run_test(test)
diff --git a/pym/portage/util/futures/unix_events.py
b/pym/portage/util/futures/unix_events.py
index d788c2bea..9d84ab6aa 100644
--- a/pym/portage/util/futures/unix_events.py
+++ b/pym/portage/util/futures/unix_events.py
@@ -9,12 +9,18 @@ __all__ = (
try:
from asyncio.base_subprocess import BaseSubprocessTransport as
_BaseSubprocessTransport
from asyncio.unix_events import AbstractChildWatcher as
_AbstractChildWatcher
+ from asyncio.transports import ReadTransport as _ReadTransport
except ImportError:
_AbstractChildWatcher = object
_BaseSubprocessTransport = object
+ _ReadTransport = object
+import errno
+import fcntl
import functools
+import logging
import os
+import stat
import subprocess
from portage.util._eventloop.global_event_loop import (
@@ -82,6 +88,35 @@ class _PortageEventLoop(events.AbstractEventLoop):
"""
return asyncio.Task(coro, loop=self)
+ def connect_read_pipe(self, protocol_factory, pipe):
+ """
+ Register read pipe in event loop. Set the pipe to non-blocking
mode.
+
+ @type protocol_factory: callable
+ @param protocol_factory: must instantiate object with Protocol
interface
+ @type pipe: file
+ @param pipe: a pipe to read from
+ @rtype: asyncio.Future
+ @return: Return pair (transport, protocol), where transport
supports the
+ ReadTransport interface.
+ """
+ protocol = protocol_factory()
+ result = self.create_future()
+ waiter = self.create_future()
+ transport = self._make_read_pipe_transport(pipe, protocol,
waiter=waiter)
+
+ def waiter_callback(waiter):
+ try:
+ waiter.result()
+ except Exception as e:
+ transport.close()
+ result.set_exception(e)
+ else:
+ result.set_result((transport, protocol))
+
+
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 2b6e90fadfb1adcd8ccd2f313aa009b3d19ffefe
Author: Zac Medico gentoo org>
AuthorDate: Sun Apr 15 23:42:49 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Apr 15 23:58:05 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2b6e90fa
EventLoop: support add_reader/writer fd overlap (bug 649588)
The AbstractEventLoop add_reader and add_writer methods need to support
simultaneous registration of reader and writer callbacks for the same
fd. For example, this feature is used by the standard library's
asyncio.unix_events._UnixWritePipeTransport class, which is used to
implement AbstractEventLoop.subprocess_exec(stdin=subprocess.PIPE).
Bug: https://bugs.gentoo.org/649588
pym/portage/util/_eventloop/EventLoop.py | 83
1 file changed, 73 insertions(+), 10 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 32dc2fc9d..35d0a35ba 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -4,6 +4,7 @@
from __future__ import division
import errno
+import functools
import logging
import os
import select
@@ -95,19 +96,20 @@ class EventLoop(object):
self._callback(*self._args)
return False
- class _repeat_callback(object):
+ class _selector_callback(object):
"""
Wraps an callback, and always returns True, for callbacks that
are supposed to run repeatedly.
"""
- __slots__ = ("_args", "_callback")
+ __slots__ = ("_args", "_callbacks")
- def __init__(self, callback, args):
- self._callback = callback
- self._args = args
+ def __init__(self, callbacks):
+ self._callbacks = callbacks
def __call__(self, fd, event):
- self._callback(*self._args)
+ for callback, mask in self._callbacks:
+ if event & mask:
+ callback()
return True
def __init__(self, main=True):
@@ -189,6 +191,9 @@ class EventLoop(object):
self.IO_OUT = PollConstants.POLLOUT
self.IO_PRI = PollConstants.POLLPRI
+ self._EVENT_READ = self.IO_IN | self.IO_HUP
+ self._EVENT_WRITE = self.IO_OUT
+
self._child_handlers = {}
self._sigchld_read = None
self._sigchld_write = None
@@ -602,7 +607,19 @@ class EventLoop(object):
Use functools.partial to pass keywords to the callback.
"""
- self.io_add_watch(fd, self.IO_IN,
self._repeat_callback(callback, args))
+ handler = self._poll_event_handlers.get(fd)
+ callbacks = [(functools.partial(callback, *args),
self._EVENT_READ)]
+ selector_mask = self._EVENT_READ
+ if handler is not None:
+ if not isinstance(handler.callback,
self._selector_callback):
+ raise AssertionError("add_reader called with fd
"
+ "registered directly via io_add_watch")
+ for item in handler.callback._callbacks:
+ callback, mask = item
+ if mask != self._EVENT_READ:
+ selector_mask |= mask
+ callbacks.append(item)
+ self.io_add_watch(fd, selector_mask,
self._selector_callback(callbacks))
def remove_reader(self, fd):
"""
@@ -610,7 +627,24 @@ class EventLoop(object):
"""
handler = self._poll_event_handlers.get(fd)
if handler is not None:
- return self.source_remove(handler.source_id)
+ if not isinstance(handler.callback,
self._selector_callback):
+ raise AssertionError("remove_reader called with
fd "
+ "registered directly via io_add_watch")
+ callbacks = []
+ selector_mask = 0
+ removed = False
+ for item in handler.callback._callbacks:
+ callback, mask = item
+ if mask == self._EVENT_READ:
+ removed = True
+ else:
+ selector_mask |= mask
+ callbacks.append(item)
+ self.source_remove(handler.source_id)
+ if callbacks:
+
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 18d8abb063d7730fbb86d451489dc2acf36c1327
Author: Zac Medico gentoo org>
AuthorDate: Sun Apr 15 22:25:03 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Apr 15 22:31:07 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=18d8abb0
EventLoop: fix AttributeError in add/remove_reader
Fixes: 24f861173ebe ("EventLoop: implement add/remove_reader/writer for asyncio
compat (bug 649588)")
pym/portage/util/_eventloop/EventLoop.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index a928f3138..32dc2fc9d 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -609,7 +609,7 @@ class EventLoop(object):
Stop watching the file descriptor for read availability.
"""
handler = self._poll_event_handlers.get(fd)
- if fd is not None:
+ if handler is not None:
return self.source_remove(handler.source_id)
return False
@@ -627,7 +627,7 @@ class EventLoop(object):
Stop watching the file descriptor for write availability.
"""
handler = self._poll_event_handlers.get(fd)
- if fd is not None:
+ if handler is not None:
return self.source_remove(handler.source_id)
return False
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/, pym/portage/util/_eventloop/
commit: 608663259a8a6fa78c32205389dfa58d00b6f11c
Author: Zac Medico gentoo org>
AuthorDate: Sun Apr 15 18:56:43 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Apr 15 18:58:00 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=60866325
Implement AbstractEventLoop.is_running() (bug 649588)
Bug: https://bugs.gentoo.org/649588
pym/portage/util/_eventloop/EventLoop.py | 14 ++
pym/portage/util/futures/unix_events.py | 1 +
2 files changed, 15 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 13ce5478e..a928f3138 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -125,6 +125,10 @@ class EventLoop(object):
self._poll_event_queue = []
self._poll_event_handlers = {}
self._poll_event_handler_ids = {}
+ # Number of current calls to self.iteration(). A number greater
+ # than 1 indicates recursion, which is not supported by
asyncio's
+ # default event loop.
+ self._iteration_depth = 0
# Increment id for each new handler.
self._event_handler_id = 0
# New call_soon callbacks must have an opportunity to
@@ -262,7 +266,13 @@ class EventLoop(object):
@rtype: bool
@return: True if events were dispatched.
"""
+ self._iteration_depth += 1
+ try:
+ return self._iteration(*args)
+ finally:
+ self._iteration_depth -= 1
+ def _iteration(self, *args):
may_block = True
if args:
@@ -822,6 +832,10 @@ class EventLoop(object):
self._default_executor = executor
return executor.submit(func, *args)
+ def is_running(self):
+ """Return whether the event loop is currently running."""
+ return self._iteration_depth > 0
+
def is_closed(self):
"""Returns True if the event loop was closed."""
return self._poll_obj is None
diff --git a/pym/portage/util/futures/unix_events.py
b/pym/portage/util/futures/unix_events.py
index 1abc420e1..d788c2bea 100644
--- a/pym/portage/util/futures/unix_events.py
+++ b/pym/portage/util/futures/unix_events.py
@@ -43,6 +43,7 @@ class _PortageEventLoop(events.AbstractEventLoop):
self.call_soon_threadsafe = loop.call_soon_threadsafe
self.call_later = loop.call_later
self.call_at = loop.call_at
+ self.is_running = loop.is_running
self.is_closed = loop.is_closed
self.close = loop.close
self.create_future = loop.create_future
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/, pym/portage/tests/util/futures/asyncio/
commit: d31db4dfb58fcd95f2590dfaed19bce4ef31bbd2
Author: Zac Medico gentoo org>
AuthorDate: Thu Apr 12 03:56:25 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Apr 13 07:10:10 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d31db4df
Implement _PortageEventLoop.subprocess_exec (bug 649588)
In python versions that support asyncio, this allows API consumers
to use the asyncio.create_subprocess_exec() function with portage's
internal event loop. Currently, subprocess.PIPE is not implemented
because that would require an implementation of asyncio's private
asyncio.unix_events._UnixReadPipeTransport class. However, it's
possible to use pipes created with os.pipe() for stdin, stdout,
and stderr, as demonstrated in the included unit tests.
Bug: https://bugs.gentoo.org/649588
.../util/futures/asyncio/test_subprocess_exec.py | 163 +
pym/portage/util/futures/unix_events.py| 98 +
2 files changed, 261 insertions(+)
diff --git a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
new file mode 100644
index 0..d30f48c43
--- /dev/null
+++ b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
@@ -0,0 +1,163 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+
+from portage.process import find_binary
+from portage.tests import TestCase
+from portage.util.futures import asyncio
+from portage.util.futures.executor.fork import ForkExecutor
+from portage.util.futures.unix_events import DefaultEventLoopPolicy
+from _emerge.PipeReader import PipeReader
+
+
+def reader(input_file, loop=None):
+ """
+ Asynchronously read a binary input file.
+
+ @param input_file: binary input file
+ @type input_file: file
+ @param loop: event loop
+ @type loop: EventLoop
+ @return: bytes
+ @rtype: asyncio.Future (or compatible)
+ """
+ loop = loop or asyncio.get_event_loop()
+ loop = getattr(loop, '_asyncio_wrapper', loop)
+ future = loop.create_future()
+ _Reader(future, input_file, loop)
+ return future
+
+
+class _Reader(object):
+ def __init__(self, future, input_file, loop):
+ self._future = future
+ self._pipe_reader = PipeReader(
+ input_files={'input_file':input_file},
scheduler=loop._loop)
+
+ self._future.add_done_callback(self._cancel_callback)
+ self._pipe_reader.addExitListener(self._eof)
+ self._pipe_reader.start()
+
+ def _cancel_callback(self, future):
+ if future.cancelled():
+ self._cancel()
+
+ def _eof(self, pipe_reader):
+ self._pipe_reader = None
+ self._future.set_result(pipe_reader.getvalue())
+
+ def _cancel(self):
+ if self._pipe_reader is not None and self._pipe_reader.poll()
is None:
+ self._pipe_reader.removeExitListener(self._eof)
+ self._pipe_reader.cancel()
+ self._pipe_reader = None
+
+
+class SubprocessExecTestCase(TestCase):
+ def _run_test(self, test):
+ initial_policy = asyncio.get_event_loop_policy()
+ if not isinstance(initial_policy, DefaultEventLoopPolicy):
+ asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
+
+ try:
+ test(asyncio.get_event_loop())
+ finally:
+ asyncio.set_event_loop_policy(initial_policy)
+
+ def testEcho(self):
+ if not hasattr(asyncio, 'create_subprocess_exec'):
+ self.skipTest('create_subprocess_exec not implemented
for python2')
+
+ args_tuple = (b'hello', b'world')
+ echo_binary = find_binary("echo")
+ self.assertNotEqual(echo_binary, None)
+ echo_binary = echo_binary.encode()
+
+ # Use os.pipe(), since this loop does not implement the
+ # ReadTransport necessary for subprocess.PIPE support.
+ stdout_pr, stdout_pw = os.pipe()
+ stdout_pr = os.fdopen(stdout_pr, 'rb', 0)
+ stdout_pw = os.fdopen(stdout_pw, 'wb', 0)
+ files = [stdout_pr, stdout_pw]
+
+ def test(loop):
+ output = None
+ try:
+ with open(os.devnull, 'rb', 0) as devnull:
+ proc = loop.run_until_complete(
+ asyncio.create_subprocess_exec(
+ echo_binary, *args_tuple,
+ stdin=devnull,
stdout=stdout_pw, stderr=stdout_pw))
+
+
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/, pym/portage/util/_eventloop/
commit: 449b7b9f30c869781f7012ca53e9cda4efef6f9b
Author: Zac Medico gentoo org>
AuthorDate: Thu Apr 12 19:28:47 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Apr 12 19:28:47 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=449b7b9f
Implement AbstractEventLoop.call_exception_handler (bug 649588)
When using asyncio with _PortageEventLoopPolicy, this method is
required in order to avoid a NotImplementedError if a coroutine
raises an unexpected exception.
Bug: https://bugs.gentoo.org/649588
pym/portage/util/_eventloop/EventLoop.py | 68
pym/portage/util/futures/unix_events.py | 2 +
2 files changed, 70 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 12c199c76..13ce5478e 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -9,6 +9,7 @@ import os
import select
import signal
import sys
+import traceback
try:
import fcntl
@@ -842,6 +843,73 @@ class EventLoop(object):
close()
self._poll_obj = None
+ def default_exception_handler(self, context):
+ """
+ Default exception handler.
+
+ This is called when an exception occurs and no exception
+ handler is set, and can be called by a custom exception
+ handler that wants to defer to the default behavior.
+
+ The context parameter has the same meaning as in
+ `call_exception_handler()`.
+
+ @param context: exception context
+ @type context: dict
+ """
+ message = context.get('message')
+ if not message:
+ message = 'Unhandled exception in event loop'
+
+ exception = context.get('exception')
+ if exception is not None:
+ exc_info = (type(exception), exception,
exception.__traceback__)
+ else:
+ exc_info = False
+
+ log_lines = [message]
+ for key in sorted(context):
+ if key in {'message', 'exception'}:
+ continue
+ value = context[key]
+ if key == 'source_traceback':
+ tb = ''.join(traceback.format_list(value))
+ value = 'Object created at (most recent call
last):\n'
+ value += tb.rstrip()
+ elif key == 'handle_traceback':
+ tb = ''.join(traceback.format_list(value))
+ value = 'Handle created at (most recent call
last):\n'
+ value += tb.rstrip()
+ else:
+ value = repr(value)
+ log_lines.append('{}: {}'.format(key, value))
+
+ logging.error('\n'.join(log_lines), exc_info=exc_info)
+ os.kill(os.getpid(), signal.SIGTERM)
+
+ def call_exception_handler(self, context):
+ """
+ Call the current event loop's exception handler.
+
+ The context argument is a dict containing the following keys:
+
+ - 'message': Error message;
+ - 'exception' (optional): Exception object;
+ - 'future' (optional): Future instance;
+ - 'handle' (optional): Handle instance;
+ - 'protocol' (optional): Protocol instance;
+ - 'transport' (optional): Transport instance;
+ - 'socket' (optional): Socket instance;
+ - 'asyncgen' (optional): Asynchronous generator that caused
+ the exception.
+
+ New keys may be introduced in the future.
+
+ @param context: exception context
+ @type context: dict
+ """
+ self.default_exception_handler(context)
+
def get_debug(self):
"""
Get the debug mode (bool) of the event loop.
diff --git a/pym/portage/util/futures/unix_events.py
b/pym/portage/util/futures/unix_events.py
index 6fcef45fa..5434cd942 100644
--- a/pym/portage/util/futures/unix_events.py
+++ b/pym/portage/util/futures/unix_events.py
@@ -48,6 +48,8 @@ class _PortageEventLoop(events.AbstractEventLoop):
self.remove_writer = loop.remove_writer
self.run_in_executor = loop.run_in_executor
self.time = loop.time
+ self.default_exception_handler = loop.default_exception_handler
+ self.call_exception_handler = loop.call_exception_handler
self.set_debug = loop.set_debug
self.get_debug = loop.get_debug
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/, pym/portage/tests/util/futures/asyncio/, ...
commit: a78dca7e47f79ad48aee4909ee10688604996b86
Author: Zac Medico gentoo org>
AuthorDate: Wed Apr 11 06:44:41 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Apr 12 08:35:05 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a78dca7e
Implement AbstractEventLoopPolicy.get_child_watcher() (bug 649588)
Use a _PortageChildWatcher class to wrap portage's internal event loop
and implement asyncio's AbstractChildWatcher interface.
Bug: https://bugs.gentoo.org/649588
.../util/futures/asyncio/test_child_watcher.py | 45 +++
pym/portage/util/_eventloop/EventLoop.py | 7 +-
pym/portage/util/futures/_asyncio.py | 13
pym/portage/util/futures/unix_events.py| 90 ++
4 files changed, 152 insertions(+), 3 deletions(-)
diff --git a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
new file mode 100644
index 0..dca01be56
--- /dev/null
+++ b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
@@ -0,0 +1,45 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+
+from portage.process import find_binary, spawn
+from portage.tests import TestCase
+from portage.util.futures import asyncio
+from portage.util.futures.unix_events import DefaultEventLoopPolicy
+
+
+class ChildWatcherTestCase(TestCase):
+ def testChildWatcher(self):
+ true_binary = find_binary("true")
+ self.assertNotEqual(true_binary, None)
+
+ initial_policy = asyncio.get_event_loop_policy()
+ if not isinstance(initial_policy, DefaultEventLoopPolicy):
+ asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
+
+ try:
+ try:
+ asyncio.set_child_watcher(None)
+ except NotImplementedError:
+ pass
+ else:
+ self.assertTrue(False)
+
+ args_tuple = ('hello', 'world')
+
+ loop = asyncio.get_event_loop()
+ future = loop.create_future()
+
+ def callback(pid, returncode, *args):
+ future.set_result((pid, returncode, args))
+
+ with asyncio.get_child_watcher() as watcher:
+ pids = spawn([true_binary], returnpid=True)
+ watcher.add_child_handler(pids[0], callback,
*args_tuple)
+
+ self.assertEqual(
+ loop.run_until_complete(future),
+ (pids[0], os.EX_OK, args_tuple))
+ finally:
+ asyncio.set_event_loop_policy(initial_policy)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index d53a76ba1..12c199c76 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -25,7 +25,7 @@ import portage
portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures.futures:Future',
'portage.util.futures.executor.fork:ForkExecutor',
- 'portage.util.futures.unix_events:_PortageEventLoop',
+
'portage.util.futures.unix_events:_PortageEventLoop,_PortageChildWatcher',
)
from portage import OrderedDict
@@ -190,6 +190,7 @@ class EventLoop(object):
self._sigchld_src_id = None
self._pid = os.getpid()
self._asyncio_wrapper = _PortageEventLoop(loop=self)
+ self._asyncio_child_watcher = _PortageChildWatcher(self)
def create_future(self):
"""
@@ -424,8 +425,8 @@ class EventLoop(object):
self._sigchld_read, self.IO_IN,
self._sigchld_io_cb)
signal.signal(signal.SIGCHLD,
self._sigchld_sig_cb)
- # poll now, in case the SIGCHLD has already arrived
- self._poll_child_processes()
+ # poll soon, in case the SIGCHLD has already arrived
+ self.call_soon(self._poll_child_processes)
return source_id
def _sigchld_sig_cb(self, signum, frame):
diff --git a/pym/portage/util/futures/_asyncio.py
b/pym/portage/util/futures/_asyncio.py
index 02ab5..0f84f14b7 100644
--- a/pym/portage/util/futures/_asyncio.py
+++ b/pym/portage/util/futures/_asyncio.py
@@ -3,7 +3,9 @@
__all__ = (
'ensure_future',
+ 'get_child_watcher',
'get_event_loop',
+ 'set_child_watcher',
'get_event_loop_policy',
'set_event_loop_policy',
'sleep',
@@ -62,6 +64,17 @@ def get_event_loop():
return get_event_loop_policy().get_event_loop()
+def
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/executor/
commit: ae59758e395393601a389ede4f4b521db6786139 Author: Zac Medico gentoo org> AuthorDate: Thu Apr 12 08:25:17 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 12 08:30:49 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ae59758e ForkExecutor: fix shutdown to handle empty self._running_tasks pym/portage/util/futures/executor/fork.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pym/portage/util/futures/executor/fork.py b/pym/portage/util/futures/executor/fork.py index 919a72bfd..51367f934 100644 --- a/pym/portage/util/futures/executor/fork.py +++ b/pym/portage/util/futures/executor/fork.py @@ -96,6 +96,8 @@ class ForkExecutor(object): def shutdown(self, wait=True): self._shutdown = True + if not self._running_tasks and not self._shutdown_future.done(): + self._shutdown_future.set_result(None) if wait: self._loop.run_until_complete(self._shutdown_future)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/executor/
commit: 52d4689740f5f9fcf1cf7423e3fe4089dbb4c718
Author: Zac Medico gentoo org>
AuthorDate: Wed Apr 11 02:01:43 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Apr 11 02:01:43 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=52d46897
ForkExecutor: support asyncio via _PortageEventLoopPolicy (bug 649588)
Support portage's internal EventLoop as well as the _PortageEventLoop
asyncio compatibility wrapper, by using the respective _loop and
_asyncio_wrapper attributes where appropriate.
Bug: https://bugs.gentoo.org/649588
pym/portage/util/futures/executor/fork.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/futures/executor/fork.py
b/pym/portage/util/futures/executor/fork.py
index 496b4e892..919a72bfd 100644
--- a/pym/portage/util/futures/executor/fork.py
+++ b/pym/portage/util/futures/executor/fork.py
@@ -25,7 +25,8 @@ class ForkExecutor(object):
"""
def __init__(self, max_workers=None, loop=None):
self._max_workers = max_workers or multiprocessing.cpu_count()
- self._loop = loop or global_event_loop()
+ loop = loop or global_event_loop()
+ self._loop = getattr(loop, '_asyncio_wrapper', loop)
self._submit_queue = collections.deque()
self._running_tasks = {}
self._shutdown = False
@@ -53,7 +54,7 @@ class ForkExecutor(object):
future, proc = self._submit_queue.popleft()
future.add_done_callback(functools.partial(self._cancel_cb, proc))
proc.addExitListener(functools.partial(self._proc_exit,
future))
- proc.scheduler = self._loop
+ proc.scheduler = self._loop._loop
proc.start()
self._running_tasks[id(proc)] = proc
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/, pym/portage/util/_eventloop/, ...
commit: 142d08c0636b172fbc00a7f2b10dc07479a57e2d Author: Zac Medico gentoo org> AuthorDate: Sun Mar 4 20:10:55 2018 + Commit: Zac Medico gentoo org> CommitDate: Wed Apr 11 01:44:34 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=142d08c0 Add minimal asyncio.AbstractEventLoop implementation (bug 649588) This provides minimal interoperability with existing asyncio code, by adding a portage.util.futures.unix_events.DefaultEventLoopPolicy class that makes asyncio use portage's internal event loop when an instance is passed into asyncio.set_event_loop_policy(). The get_event_loop() method of this policy returns an instance of a _PortageEventLoop class that wraps portage's internal event loop and implements asyncio's AbstractEventLoop interface. The portage.util.futures.asyncio module refers to the real asyncio module when available, and otherwise falls back to a minimal implementation that works with python2.7. The included EventLoopInForkTestCase demonstrates usage, and works with all supported versions of python, include python2.7. In python3.4 and later, API consumers can use asyncio coroutines, since _PortageEventLoop is compatible with asyncio.Task! Bug: https://bugs.gentoo.org/649588 .../util/futures/asyncio}/__init__.py | 0 .../util/futures/asyncio/__test__.py} | 0 .../futures/asyncio/test_event_loop_in_fork.py | 59 +++ pym/portage/util/_eventloop/EventLoop.py | 11 +- pym/portage/util/futures/__init__.py | 11 ++ pym/portage/util/futures/_asyncio.py | 114 pym/portage/util/futures/events.py | 191 + pym/portage/util/futures/futures.py| 9 +- pym/portage/util/futures/unix_events.py| 91 ++ 9 files changed, 477 insertions(+), 9 deletions(-) diff --git a/pym/portage/util/futures/__init__.py b/pym/portage/tests/util/futures/asyncio/__init__.py similarity index 100% copy from pym/portage/util/futures/__init__.py copy to pym/portage/tests/util/futures/asyncio/__init__.py diff --git a/pym/portage/util/futures/__init__.py b/pym/portage/tests/util/futures/asyncio/__test__.py similarity index 100% copy from pym/portage/util/futures/__init__.py copy to pym/portage/tests/util/futures/asyncio/__test__.py diff --git a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py new file mode 100644 index 0..7868d792a --- /dev/null +++ b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py @@ -0,0 +1,59 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import multiprocessing +import os + +from portage.tests import TestCase +from portage.util.futures import asyncio +from portage.util.futures.unix_events import DefaultEventLoopPolicy + + +def fork_main(parent_conn, child_conn): + parent_conn.close() + loop = asyncio.get_event_loop() + # This fails with python's default event loop policy, + # see https://bugs.python.org/issue22087. + loop.run_until_complete(asyncio.sleep(0.1)) + + +def async_main(fork_exitcode, loop=None): + loop = loop or asyncio.get_event_loop() + + # Since python2.7 does not support Process.sentinel, use Pipe to + # monitor for process exit. + parent_conn, child_conn = multiprocessing.Pipe() + + def eof_callback(proc): + loop.remove_reader(parent_conn.fileno()) + parent_conn.close() + proc.join() + fork_exitcode.set_result(proc.exitcode) + + proc = multiprocessing.Process(target=fork_main, args=(parent_conn, child_conn)) + loop.add_reader(parent_conn.fileno(), eof_callback, proc) + proc.start() + child_conn.close() + + +class EventLoopInForkTestCase(TestCase): + """ + The default asyncio event loop policy does not support loops + running in forks, see https://bugs.python.org/issue22087. + Portage's DefaultEventLoopPolicy supports forks. + """ + + def testEventLoopInForkTestCase(self): + initial_policy = asyncio.get_event_loop_policy() + if not isinstance(initial_policy, DefaultEventLoopPolicy): + asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) + try: + loop = asyncio.get_event_loop() + fork_exitcode = loop.create_future() + # Make async_main fork while the loop is running, which would + # trigger https://bugs.python.org/issue22087 with asyncio's + # default event loop policy. + loop.call_soon(async_main, fork_exitcode) + assert loop.run_until_complete(fork_exitcode) == os.EX_OK + finally: +
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 3b1c182750d82dc55b96ee111e356968ca9a9fb7 Author: Zac Medico gentoo org> AuthorDate: Mon Apr 9 01:25:25 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 9 01:25:52 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3b1c1827 EventLoop: add call_at method for asyncio compat (bug 591760) Bug: https://bugs.gentoo.org/591760 pym/portage/util/_eventloop/EventLoop.py | 23 +++ 1 file changed, 23 insertions(+) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 00568c997..72eb407fc 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -775,6 +775,29 @@ class EventLoop(object): return self._handle(self.timeout_add( delay * 1000, self._call_soon_callback(callback, args)), self) + def call_at(self, when, callback, *args): + """ + Arrange for the callback to be called at the given absolute + timestamp when (an int or float), using the same time reference as + AbstractEventLoop.time(). + + This method's behavior is the same as call_later(). + + An instance of asyncio.Handle is returned, which can be used to + cancel the callback. + + Use functools.partial to pass keywords to the callback. + + @type when: int or float + @param when: absolute timestamp when to call callback + @type callback: callable + @param callback: a function to call + @return: a handle which can be used to cancel the callback + @rtype: asyncio.Handle (or compatible) + """ + delta = when - self.time() + return self.call_later(delta if delta > 0 else 0, callback, *args) + def run_in_executor(self, executor, func, *args): """ Arrange for a func to be called in the specified executor.
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 24f861173ebe747a470deb8489887c067cd46b0f
Author: Zac Medico gentoo org>
AuthorDate: Mon Apr 2 03:46:10 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Apr 8 22:04:37 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=24f86117
EventLoop: implement add/remove_reader/writer for asyncio compat (bug 649588)
Bug: https://bugs.gentoo.org/649588
pym/portage/util/_eventloop/EventLoop.py | 51
1 file changed, 51 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 1bf606354..00568c997 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -93,6 +93,21 @@ class EventLoop(object):
self._callback(*self._args)
return False
+ class _repeat_callback(object):
+ """
+ Wraps an callback, and always returns True, for callbacks that
+ are supposed to run repeatedly.
+ """
+ __slots__ = ("_args", "_callback")
+
+ def __init__(self, callback, args):
+ self._callback = callback
+ self._args = args
+
+ def __call__(self, fd, event):
+ self._callback(*self._args)
+ return True
+
def __init__(self, main=True):
"""
@param main: If True then this is a singleton instance for use
@@ -569,6 +584,42 @@ class EventLoop(object):
return bool(calls)
+ def add_reader(self, fd, callback, *args):
+ """
+ Start watching the file descriptor for read availability and
then
+ call the callback with specified arguments.
+
+ Use functools.partial to pass keywords to the callback.
+ """
+ self.io_add_watch(fd, self.IO_IN,
self._repeat_callback(callback, args))
+
+ def remove_reader(self, fd):
+ """
+ Stop watching the file descriptor for read availability.
+ """
+ handler = self._poll_event_handlers.get(fd)
+ if fd is not None:
+ return self.source_remove(handler.source_id)
+ return False
+
+ def add_writer(self, fd, callback, *args):
+ """
+ Start watching the file descriptor for write availability and
then
+ call the callback with specified arguments.
+
+ Use functools.partial to pass keywords to the callback.
+ """
+ self.io_add_watch(fd, self.IO_OUT,
self._repeat_callback(callback, args))
+
+ def remove_writer(self, fd):
+ """
+ Stop watching the file descriptor for write availability.
+ """
+ handler = self._poll_event_handlers.get(fd)
+ if fd is not None:
+ return self.source_remove(handler.source_id)
+ return False
+
def io_add_watch(self, f, condition, callback, *args):
"""
Like glib.io_add_watch(), your function should return False to
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 754010f346ec2455ea8c71a6af4796c10fd28d23 Author: Zac Medico gentoo org> AuthorDate: Sun Apr 8 21:36:07 2018 + Commit: Zac Medico gentoo org> CommitDate: Sun Apr 8 21:36:07 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=754010f3 EventLoop: add is_closed method for asyncio compat (bug 591760) Bug: https://bugs.gentoo.org/591760 pym/portage/util/_eventloop/EventLoop.py | 4 1 file changed, 4 insertions(+) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 4ec67241f..1bf606354 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -747,6 +747,10 @@ class EventLoop(object): self._default_executor = executor return executor.submit(func, *args) + def is_closed(self): + """Returns True if the event loop was closed.""" + return self._poll_obj is None + def close(self): """Close the event loop.
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: 13621b62e32a7c21aa08247b33f1faa0a146d0d4
Author: Zac Medico gentoo org>
AuthorDate: Sun Apr 8 21:15:39 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Apr 8 21:15:39 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=13621b62
EventLoop: add get/set_debug methods for asyncio compat (bug 591760)
Bug: https://bugs.gentoo.org/591760
pym/portage/util/_eventloop/EventLoop.py | 14 ++
1 file changed, 14 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 1574a6837..4ec67241f 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -102,6 +102,7 @@ class EventLoop(object):
@type main: bool
"""
self._use_signal = main and fcntl is not None
+ self._debug = bool(os.environ.get('PYTHONASYNCIODEBUG'))
self._thread_rlock = threading.RLock()
self._thread_condition = threading.Condition(self._thread_rlock)
self._poll_event_queue = []
@@ -763,6 +764,19 @@ class EventLoop(object):
close()
self._poll_obj = None
+ def get_debug(self):
+ """
+ Get the debug mode (bool) of the event loop.
+
+ The default value is True if the environment variable
+ PYTHONASYNCIODEBUG is set to a non-empty string, False
otherwise.
+ """
+ return self._debug
+
+ def set_debug(self, enabled):
+ """Set the debug mode of the event loop."""
+ self._debug = enabled
+
_can_poll_device = None
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: db3d216676831d8b6c184fb25f94dc54f8710a11 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 7 20:19:26 2018 + Commit: Zac Medico gentoo org> CommitDate: Sat Apr 7 20:20:15 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=db3d2166 GlibEventLoop: remove (unused and unmaintained) pym/portage/util/_eventloop/GlibEventLoop.py | 23 --- pym/portage/util/_eventloop/global_event_loop.py | 1 - 2 files changed, 24 deletions(-) diff --git a/pym/portage/util/_eventloop/GlibEventLoop.py b/pym/portage/util/_eventloop/GlibEventLoop.py deleted file mode 100644 index f2f5c5e64..0 --- a/pym/portage/util/_eventloop/GlibEventLoop.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -class GlibEventLoop(object): - - # TODO: Support multiprocessing by using a separate glib.MainContext - # instance for each process. - supports_multiprocessing = False - - def __init__(self): - import gi.repository.GLib as glib - self.IO_ERR = glib.IO_ERR - self.IO_HUP = glib.IO_HUP - self.IO_IN = glib.IO_IN - self.IO_NVAL = glib.IO_NVAL - self.IO_OUT = glib.IO_OUT - self.IO_PRI = glib.IO_PRI - self.iteration = glib.main_context_default().iteration - self.child_watch_add = glib.child_watch_add - self.idle_add = glib.idle_add - self.io_add_watch = glib.io_add_watch - self.timeout_add = glib.timeout_add - self.source_remove = glib.source_remove diff --git a/pym/portage/util/_eventloop/global_event_loop.py b/pym/portage/util/_eventloop/global_event_loop.py index 502dab882..e2c7d71ea 100644 --- a/pym/portage/util/_eventloop/global_event_loop.py +++ b/pym/portage/util/_eventloop/global_event_loop.py @@ -6,7 +6,6 @@ import os from .EventLoop import EventLoop _default_constructor = EventLoop -#from .GlibEventLoop import GlibEventLoop as _default_constructor # If _default_constructor doesn't support multiprocessing, # then _multiprocessing_constructor is used in subprocesses.
[gentoo-commits] proj/portage:master commit in: pym/portage/util/elf/
commit: 8443230a7865ca3ce006023054d4313f680c9508 Author: Arfrever Frehtes Taifersar Arahesis Apache Org> AuthorDate: Thu Apr 5 23:48:03 2018 + Commit: Zac Medico gentoo org> CommitDate: Thu Apr 5 23:55:04 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8443230a portage.util.elf.constants: Update URL for elfutils header. fedorahosted.org was retired on 2017-03-01. pym/portage/util/elf/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/elf/constants.py b/pym/portage/util/elf/constants.py index 4761cf427..2704e85c3 100644 --- a/pym/portage/util/elf/constants.py +++ b/pym/portage/util/elf/constants.py @@ -1,8 +1,8 @@ -# Copyright 2015 Gentoo Foundation +# Copyright 2015-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # # These constants are available from elfutils: -# https://git.fedorahosted.org/cgit/elfutils.git/tree/libelf/elf.h +# https://sourceware.org/git/?p=elfutils.git;a=blob;f=libelf/elf.h;hb=HEAD EI_CLASS = 4 ELFCLASS32 = 1
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 6d822cbc2a7a68e39583991b3f69ff76b032d585 Author: Zac Medico gentoo org> AuthorDate: Mon Mar 19 07:37:38 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 2 16:53:23 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6d822cbc Add ExponentialBackoff and RandomExponentialBackoff This will be useful as parameters for retry decorators. pym/portage/util/backoff.py | 53 + 1 file changed, 53 insertions(+) diff --git a/pym/portage/util/backoff.py b/pym/portage/util/backoff.py new file mode 100644 index 0..ee39007ef --- /dev/null +++ b/pym/portage/util/backoff.py @@ -0,0 +1,53 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +__all__ = ( + 'ExponentialBackoff', + 'RandomExponentialBackoff', +) + +import random +import sys + + +class ExponentialBackoff(object): + """ + An object that when called with number of previous tries, calculates + an exponential delay for the next try. + """ + def __init__(self, multiplier=1, base=2, limit=sys.maxsize): + """ + @param multiplier: constant multiplier + @type multiplier: int or float + @param base: maximum number of tries + @type base: int or float + @param limit: maximum number of seconds to delay + @type limit: int or float + """ + self._multiplier = multiplier + self._base = base + self._limit = limit + + def __call__(self, tries): + """ + Given a number of previous tries, calculate the amount of time + to delay the next try. + + @param tries: number of previous tries + @type tries: int + @return: amount of time to delay the next try + @rtype: int + """ + try: + return min(self._limit, self._multiplier * (self._base ** tries)) + except OverflowError: + return self._limit + + +class RandomExponentialBackoff(ExponentialBackoff): + """ + Equivalent to ExponentialBackoff, with an extra multiplier that uses + a random distribution between 0 and 1. + """ + def __call__(self, tries): + return random.random() * super(RandomExponentialBackoff, self).__call__(tries)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/executor/, pym/portage/util/_eventloop/
commit: 4095be74985c5c2eead5fb480cf37baa11308d62
Author: Zac Medico gentoo org>
AuthorDate: Wed Mar 14 08:01:26 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Apr 2 16:53:23 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4095be74
Add ForkExecutor (bug 649588)
This is useful for asynchronous operations that we might
need to cancel if they take too long, since (concurrent.
futures.ProcessPoolExecutor tasks are not cancellable).
The ability to cancel tasks makes this executor useful
as an alternative to portage.exception.AlarmSignal.
Also add an asyncio-compatible EventLoop.run_in_executor
method that uses ForkExecutor as the default executor,
which will later be used to implement the corresponding
asyncio.AbstractEventLoop run_in_executor method.
Bug: https://bugs.gentoo.org/649588
Reviewed-by: Alec Warner gentoo.org>
pym/portage/util/_eventloop/EventLoop.py | 45 -
pym/portage/util/futures/executor/__init__.py | 0
pym/portage/util/futures/executor/fork.py | 134 ++
3 files changed, 178 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index f472a3dae..1574a6837 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -24,6 +24,7 @@ except ImportError:
import portage
portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures.futures:_EventLoopFuture',
+ 'portage.util.futures.executor.fork:ForkExecutor',
)
from portage import OrderedDict
@@ -122,6 +123,7 @@ class EventLoop(object):
self._idle_callbacks = OrderedDict()
self._timeout_handlers = {}
self._timeout_interval = None
+ self._default_executor = None
self._poll_obj = None
try:
@@ -721,6 +723,46 @@ class EventLoop(object):
return self._handle(self.timeout_add(
delay * 1000, self._call_soon_callback(callback,
args)), self)
+ def run_in_executor(self, executor, func, *args):
+ """
+ Arrange for a func to be called in the specified executor.
+
+ The executor argument should be an Executor instance. The
default
+ executor is used if executor is None.
+
+ Use functools.partial to pass keywords to the *func*.
+
+ @param executor: executor
+ @type executor: concurrent.futures.Executor or None
+ @param func: a function to call
+ @type func: callable
+ @return: a Future
+ @rtype: asyncio.Future (or compatible)
+ """
+ if executor is None:
+ executor = self._default_executor
+ if executor is None:
+ executor = ForkExecutor(loop=self)
+ self._default_executor = executor
+ return executor.submit(func, *args)
+
+ def close(self):
+ """Close the event loop.
+
+ This clears the queues and shuts down the executor,
+ and waits for it to finish.
+ """
+ executor = self._default_executor
+ if executor is not None:
+ self._default_executor = None
+ executor.shutdown(wait=True)
+
+ if self._poll_obj is not None:
+ close = getattr(self._poll_obj, 'close')
+ if close is not None:
+ close()
+ self._poll_obj = None
+
_can_poll_device = None
@@ -782,10 +824,11 @@ class _epoll_adapter(object):
that is associated with an epoll instance will close automatically when
it is garbage collected, so it's not necessary to close it explicitly.
"""
- __slots__ = ('_epoll_obj',)
+ __slots__ = ('_epoll_obj', 'close')
def __init__(self, epoll_obj):
self._epoll_obj = epoll_obj
+ self.close = epoll_obj.close
def register(self, fd, *args):
self._epoll_obj.register(fd, *args)
diff --git a/pym/portage/util/futures/executor/__init__.py
b/pym/portage/util/futures/executor/__init__.py
new file mode 100644
index 0..e69de29bb
diff --git a/pym/portage/util/futures/executor/fork.py
b/pym/portage/util/futures/executor/fork.py
new file mode 100644
index 0..496b4e892
--- /dev/null
+++ b/pym/portage/util/futures/executor/fork.py
@@ -0,0 +1,134 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+__all__ = (
+ 'ForkExecutor',
+)
+
+import collections
+import functools
+import multiprocessing
+import os
+import sys
+import traceback
+
+from portage.util._async.AsyncFunction import
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 8f9cff73321b3ee5957357448819b505932e0e5c
Author: Michał Górny gentoo org>
AuthorDate: Sun May 22 06:40:13 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Mar 28 06:28:17 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8f9cff73
{,PKG_}INSTALL_MASK: Support exclusions (bug 651214)
Allow INSTALL_MASK patterns to start with '-' to indicate that
a specific match is to be excluded from being masked. In this case,
the last matching pattern determines whether the file is actually
filtered out or kept.
Bug: https://bugs.gentoo.org/651214
pym/portage/util/install_mask.py | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/install_mask.py b/pym/portage/util/install_mask.py
index 506e63c1f..1667d883a 100644
--- a/pym/portage/util/install_mask.py
+++ b/pym/portage/util/install_mask.py
@@ -35,19 +35,21 @@ class InstallMask(object):
"""
ret = False
for pattern in self._install_mask:
+ # if pattern starts with -, possibly exclude this path
+ is_inclusive = not pattern.startswith('-')
+ if not is_inclusive:
+ pattern = pattern[1:]
# absolute path pattern
if pattern.startswith('/'):
# match either exact path or one of parent dirs
# the latter is done via matching pattern/*
if (fnmatch.fnmatch(path, pattern[1:])
or fnmatch.fnmatch(path,
pattern[1:] + '/*')):
- ret = True
- break
+ ret = is_inclusive
# filename
else:
if fnmatch.fnmatch(os.path.basename(path),
pattern):
- ret = True
- break
+ ret = is_inclusive
return ret
[gentoo-commits] proj/portage:master commit in: pym/portage/util/, pym/portage/util/_eventloop/
commit: a997f3e2d4318b8b9129d5cccd3c5f2c5677d11d Author: Zac Medico gentoo org> AuthorDate: Fri Mar 16 08:40:42 2018 + Commit: Zac Medico gentoo org> CommitDate: Fri Mar 16 17:17:53 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a997f3e2 EventLoop: implement time method for asyncio compat (bug 591760) Use time.monotonic() which is available in Python 3.3 and later, and otherwise emulate it by using an offset to counteract any backward movements. Bug: https://bugs.gentoo.org/591760 pym/portage/util/_eventloop/EventLoop.py | 19 +- pym/portage/util/monotonic.py| 34 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 89ac2a3b3..f472a3dae 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -1,4 +1,4 @@ -# Copyright 1999-2016 Gentoo Foundation +# Copyright 1999-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import division @@ -9,7 +9,6 @@ import os import select import signal import sys -import time try: import fcntl @@ -29,6 +28,7 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import OrderedDict from portage.util import writemsg_level +from portage.util.monotonic import monotonic from ..SlotObject import SlotObject from .PollConstants import PollConstants from .PollSelectAdapter import PollSelectAdapter @@ -515,7 +515,7 @@ class EventLoop(object): self._timeout_handlers[source_id] = \ self._timeout_handler_class( interval=interval, function=function, args=args, - source_id=source_id, timestamp=time.time()) + source_id=source_id, timestamp=self.time()) if self._timeout_interval is None or \ self._timeout_interval > interval: self._timeout_interval = interval @@ -538,7 +538,7 @@ class EventLoop(object): return bool(calls) ready_timeouts = [] - current_time = time.time() + current_time = self.time() for x in self._timeout_handlers.values(): elapsed_seconds = current_time - x.timestamp # elapsed_seconds < 0 means the system clock has been adjusted @@ -558,7 +558,7 @@ class EventLoop(object): calls += 1 x.calling = True try: - x.timestamp = time.time() + x.timestamp = self.time() if not x.function(*x.args): self.source_remove(x.source_id) finally: @@ -684,6 +684,15 @@ class EventLoop(object): # The call_soon method inherits thread safety from the idle_add method. call_soon_threadsafe = call_soon + def time(self): + """Return the time according to the event loop's clock. + + This is a float expressed in seconds since an epoch, but the + epoch, precision, accuracy and drift are unspecified and may + differ per event loop. + """ + return monotonic() + def call_later(self, delay, callback, *args): """ Arrange for the callback to be called after the given delay seconds diff --git a/pym/portage/util/monotonic.py b/pym/portage/util/monotonic.py new file mode 100644 index 0..e50564851 --- /dev/null +++ b/pym/portage/util/monotonic.py @@ -0,0 +1,34 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +__all__ = ['monotonic'] + +import time +try: + import threading +except ImportError: + import dummy_threading as threading + +monotonic = getattr(time, 'monotonic', None) + +if monotonic is None: + def monotonic(): + """ + Emulate time.monotonic() which is available in Python 3.3 and later. + + @return: A float expressed in seconds since an epoch. + """ + with monotonic._lock: + current = time.time() + monotonic._offset + delta = current - monotonic._previous + if delta < 0: + monotonic._offset -= delta + current = monotonic._previous + else: +
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/
commit: 71c59145e0c7b631ec3f41e0d711445786d16f8f
Author: Zac Medico gentoo org>
AuthorDate: Thu Mar 15 02:45:31 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Mar 15 02:49:02 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=71c59145
portage.util.futures.wait: fix arguments for asyncio compat
The bare "*" is not supported in python2.7, and in python3
the bare "*" means that keyword arguments must be used for
the arguments that follow.
Fixes: e43f6c583ed9 ("Add iter_completed convenience function (bug 648790)")
pym/portage/util/futures/iter_completed.py | 2 +-
pym/portage/util/futures/wait.py | 16 +++-
2 files changed, 4 insertions(+), 14 deletions(-)
diff --git a/pym/portage/util/futures/iter_completed.py
b/pym/portage/util/futures/iter_completed.py
index 1050b6fa7..ad6275b49 100644
--- a/pym/portage/util/futures/iter_completed.py
+++ b/pym/portage/util/futures/iter_completed.py
@@ -52,7 +52,7 @@ def iter_completed(futures, max_jobs=None, max_load=None,
loop=None):
# task_generator is exhausted
while future_map:
done, pending = loop.run_until_complete(
- wait(*list(future_map.values()),
return_when=FIRST_COMPLETED))
+ wait(list(future_map.values()),
return_when=FIRST_COMPLETED))
for future in done:
del future_map[id(future)]
yield future
diff --git a/pym/portage/util/futures/wait.py b/pym/portage/util/futures/wait.py
index 3f0bdbff5..bd85bb053 100644
--- a/pym/portage/util/futures/wait.py
+++ b/pym/portage/util/futures/wait.py
@@ -11,15 +11,13 @@ except ImportError:
from portage.util._eventloop.global_event_loop import global_event_loop
-# Use **kwargs since python2.7 does not allow arguments with defaults
-# to follow *futures.
-def wait(*futures, **kwargs):
+def wait(futures, loop=None, timeout=None, return_when=ALL_COMPLETED):
"""
Use portage's internal EventLoop to emulate asyncio.wait:
https://docs.python.org/3/library/asyncio-task.html#asyncio.wait
- @param future: future to wait for
- @type future: asyncio.Future (or compatible)
+ @param futures: futures to wait for
+ @type futures: asyncio.Future (or compatible)
@param timeout: number of seconds to wait (wait indefinitely if
not specified)
@type timeout: int or float
@@ -32,14 +30,6 @@ def wait(*futures, **kwargs):
@return: tuple of (done, pending).
@rtype: asyncio.Future (or compatible)
"""
- if not futures:
- raise TypeError("wait() missing 1 required positional argument:
'future'")
- loop = kwargs.pop('loop', None)
- timeout = kwargs.pop('timeout', None)
- return_when = kwargs.pop('return_when', ALL_COMPLETED)
- if kwargs:
- raise TypeError("wait() got an unexpected keyword argument
'{}'".\
- format(next(iter(kwargs
loop = loop or global_event_loop()
result_future = loop.create_future()
_Waiter(futures, timeout, return_when, result_future, loop)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: ad72389ed8067d740dd0e3f975a33f0c1cbfd8b5
Author: Arfrever Frehtes Taifersar Arahesis Apache Org>
AuthorDate: Sun Mar 4 02:25:34 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 4 02:37:09 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ad72389e
movefile: Fix preservation of security.capability xattr
Call _apply_stat() before copying extended attributes, because
_apply_stat() calls os.chown() which results in deleting
security.capability extended attribute.
Bug: https://bugs.gentoo.org/649418
pym/portage/util/movefile.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/movefile.py b/pym/portage/util/movefile.py
index 37c809eb5..5477a669f 100644
--- a/pym/portage/util/movefile.py
+++ b/pym/portage/util/movefile.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2013 Gentoo Foundation
+# Copyright 2010-2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from __future__ import absolute_import, unicode_literals
@@ -272,6 +272,7 @@ def movefile(src, dest, newmtime=None, sstat=None,
mysettings=None,
errors='strict')
try: # For safety copy then move it over.
_copyfile(src_bytes, dest_tmp_bytes)
+ _apply_stat(sstat, dest_tmp_bytes)
if xattr_enabled:
try:
_copyxattr(src_bytes,
dest_tmp_bytes,
@@ -286,7 +287,6 @@ def movefile(src, dest, newmtime=None, sstat=None,
mysettings=None,
for line in msg:
writemsg("!!! %s\n" %
(line,), noiselevel=-1)
raise
- _apply_stat(sstat, dest_tmp_bytes)
_rename(dest_tmp_bytes, dest_bytes)
_os.unlink(src_bytes)
except SystemExit as e:
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
commit: b9fc8e55f96c17aeece87387226ada1b184d2f77
Author: Zac Medico gentoo org>
AuthorDate: Tue Jan 2 09:09:34 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Jan 2 23:02:39 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9fc8e55
PreservedLibsRegistry: fix pruneNonExisting for symlinks to other dirs (bug
642672)
Fix pruneNonExisting to use the abssymlink function to detect symlinks
in the registry that no longer point to a preserved library. The previous
code only worked correctly for symlinks pointing to files in the same
directory, which failed for packages like dev-ada/xmlada which have
symlinks that point into a subdirectory.
Fixes: 32d19be14e22 ("pruneNonExisting: handle eselect-opengl symlinks")
Tested-by: Tupone Alfredo gentoo.org>
Bug: https://bugs.gentoo.org/642672
pym/portage/util/_dyn_libs/PreservedLibsRegistry.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
index a422ffefd..f83b82a31 100644
--- a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
+++ b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
@@ -12,6 +12,7 @@ try:
except ImportError:
import pickle
+from portage import abssymlink
from portage import os
from portage import _encodings
from portage import _os_merge
@@ -227,7 +228,7 @@ class PreservedLibsRegistry(object):
# removed by _remove_preserved_libs, it calls
pruneNonExisting
# which eliminates the irrelevant symlink from the
registry here.
for f, target in symlinks.items():
- if os.path.join(os.path.dirname(f), target) in
hardlinks:
+ if abssymlink(f, target=target) in hardlinks:
paths.append(f)
if len(paths) > 0:
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 78273a404ea6244eab4180dff2bd1847609ac09a Author: Zac Medico gentoo org> AuthorDate: Mon Aug 21 16:10:57 2017 + Commit: Zac Medico gentoo org> CommitDate: Mon Aug 21 16:16:05 2017 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=78273a40 digraph: search for relevant priority in descending order It's more likely that a higher valued priority will not be ignored, therefore search in descending order. pym/portage/util/digraph.py | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pym/portage/util/digraph.py b/pym/portage/util/digraph.py index bf20d5d4e..d279b7867 100644 --- a/pym/portage/util/digraph.py +++ b/pym/portage/util/digraph.py @@ -170,7 +170,7 @@ class digraph(object): children = [] if hasattr(ignore_priority, '__call__'): for child, priorities in self.nodes[node][0].items(): - for priority in priorities: + for priority in reversed(priorities): if not ignore_priority(priority): children.append(child) break @@ -187,7 +187,7 @@ class digraph(object): parents = [] if hasattr(ignore_priority, '__call__'): for parent, priorities in self.nodes[node][1].items(): - for priority in priorities: + for priority in reversed(priorities): if not ignore_priority(priority): parents.append(parent) break @@ -212,7 +212,7 @@ class digraph(object): for node in self.order: is_leaf_node = True for child, priorities in self.nodes[node][0].items(): - for priority in priorities: + for priority in reversed(priorities): if not ignore_priority(priority): is_leaf_node = False break @@ -246,7 +246,7 @@ class digraph(object): for node in self.order: is_root_node = True for parent, priorities in self.nodes[node][1].items(): - for priority in priorities: + for priority in reversed(priorities): if not ignore_priority(priority): is_root_node = False break
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 6098513ec399bd6949d6ae852be9ee83dcb24a2f Author: Zac Medico gentoo org> AuthorDate: Sun Aug 20 21:30:22 2017 + Commit: Zac Medico gentoo org> CommitDate: Mon Aug 21 15:58:42 2017 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6098513e digraph.add: sort priorities with bisect.insort Reported-by: Sergei Trofimovich gentoo.org> Reviewed-by: Manuel Rüger gentoo.org> pym/portage/util/digraph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pym/portage/util/digraph.py b/pym/portage/util/digraph.py index ba0e81c07..bf20d5d4e 100644 --- a/pym/portage/util/digraph.py +++ b/pym/portage/util/digraph.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals __all__ = ['digraph'] +import bisect from collections import deque import sys @@ -46,8 +47,7 @@ class digraph(object): self.nodes[parent][0][node] = priorities if not priorities or priorities[-1] is not priority: - priorities.append(priority) - priorities.sort() + bisect.insort(priorities, priority) def discard(self, node): """
[gentoo-commits] proj/portage:master commit in: pym/portage/util/, pym/portage/dbapi/
commit: 1d821469d6b72ce051b02908f17302c500945788
Author: Zac Medico gentoo org>
AuthorDate: Wed Jul 19 07:25:05 2017 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jul 31 16:30:05 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1d821469
emerge --getbinpkg: https support for If-Modified-Since
When https certificate and hostname verification is enabled for
stdlib http clients (PEP 476), use python for If-Modified-Since
header support. When python lacks PEP 476 support, continue to
use FETCHCOMMAND for https certificate and hostname verification
(see security bug 469888).
X-Gentoo-bug: 625246
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=625246
Acked-by: Brian Dolbec gentoo.org>
pym/portage/dbapi/bintree.py | 10 ++
pym/portage/util/_urlopen.py | 12
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index c833968c2..95bd5dbf8 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -18,7 +18,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \
'writemsg,writemsg_stdout',
'portage.util.path:first_existing',
- 'portage.util._urlopen:urlopen@_urlopen',
+ 'portage.util._urlopen:urlopen@_urlopen,have_pep_476@_have_pep_476',
'portage.versions:best,catpkgsplit,catsplit,_pkg_str',
)
@@ -851,9 +851,9 @@ class binarytree(object):
download_timestamp + ttl >
time.time():
raise
UseCachedCopyOfRemoteIndex()
- # Don't use urlopen for https, since it doesn't
support
- # certificate/hostname verification (bug
#469888).
- if parsed_url.scheme not in ('https',):
+ # Don't use urlopen for https, unless
+ # PEP 476 is supported (bug #469888).
+ if parsed_url.scheme not in ('https',) or
_have_pep_476():
try:
f = _urlopen(url,
if_modified_since=local_timestamp)
if hasattr(f, 'headers') and
f.headers.get('timestamp', ''):
@@ -965,6 +965,8 @@ class binarytree(object):
"\n")
rmt_idx = pkgindex
except EnvironmentError as e:
+ # This includes URLError which is raised for SSL
+ # certificate errors when PEP 476 is supported.
writemsg(_("\n\n!!! Error fetching binhost
package" \
" info from '%s'\n") %
_hide_url_passwd(base_url))
# With Python 2, the EnvironmentError message
may
diff --git a/pym/portage/util/_urlopen.py b/pym/portage/util/_urlopen.py
index 4cfe183b1..fc9db74a0 100644
--- a/pym/portage/util/_urlopen.py
+++ b/pym/portage/util/_urlopen.py
@@ -26,6 +26,18 @@ if sys.hexversion >= 0x300:
# and the file-'mtime'
TIMESTAMP_TOLERANCE = 5
+
+def have_pep_476():
+ """
+ Test whether ssl certificate verification is enabled by default for
+ stdlib http clients (PEP 476).
+
+ @returns: bool, True if ssl certificate verification is enabled by
+ default
+ """
+ return hasattr(__import__('ssl'), '_create_unverified_context')
+
+
def urlopen(url, if_modified_since=None):
parse_result = urllib_parse.urlparse(url)
if parse_result.scheme not in ("http", "https"):
[gentoo-commits] proj/portage:master commit in: pym/portage/util/, pym/portage/package/ebuild/, man/, ...
commit: 2c7d38b9512609f6828cbba1066f2b3b2d9144bf
Author: Zac Medico gentoo org>
AuthorDate: Sun Jul 30 20:36:14 2017 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Jul 30 20:49:40 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2c7d38b9
Rename BINPKG_COMPRESSION{,_ARGS} to BINPKG_COMPRESS{,_FLAGS}
This is more consistent with the names of the existing
PORTAGE_COMPRESS* variables.
Suggested-by: Michał Górny gentoo.org>
bin/quickpkg | 2 +-
man/make.conf.5| 10 +-
pym/portage/package/ebuild/_config/special_env_vars.py | 1 +
pym/portage/package/ebuild/config.py | 8
pym/portage/package/ebuild/doebuild.py | 2 +-
pym/portage/util/compression_probe.py | 14 +++---
6 files changed, 19 insertions(+), 18 deletions(-)
diff --git a/bin/quickpkg b/bin/quickpkg
index 750400592..392e9da22 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -137,7 +137,7 @@ def quickpkg_atom(options, infos, arg, eout):
binpkg_tmpfile = os.path.join(bintree.pkgdir,
cpv + ".tbz2." + str(os.getpid()))
ensure_dirs(os.path.dirname(binpkg_tmpfile))
- binpkg_compression = settings.get("BINPKG_COMPRESSION",
"bzip2")
+ binpkg_compression = settings.get("BINPKG_COMPRESS",
"bzip2")
try:
compression = _compressors[binpkg_compression]
except KeyError as e:
diff --git a/man/make.conf.5 b/man/make.conf.5
index 8e0d04967..65c18cc6d 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -110,7 +110,7 @@ ACCEPT_RESTRICT="*"
ACCEPT_RESTRICT="* -bindist"
.fi
.TP
-\fBBINPKG_COMPRESSION\fR = \fI"compression"\fR
+\fBBINPKG_COMPRESS\fR = \fI"compression"\fR
This variable is used to determine the compression used for \fIbinary
packages\fR. Supported settings and compression algorithms are: bzip2, gzip,
lz4, lzip, lzop, xz, zstd.
@@ -120,19 +120,19 @@ Defaults to "bzip2".
.I Example:
.nf
# Set it to use lz4:
-BINPKG_COMPRESSION="lz4"
+BINPKG_COMPRESS="lz4"
.fi
.TP
-\fBBINPKG_COMPRESSION_ARGS\fR = \fI"arguments for compression command"\fR
+\fBBINPKG_COMPRESS_FLAGS\fR = \fI"arguments for compression command"\fR
This variable is used to add additional arguments to the compression command
-selected by \fBBINPKG_COMPRESSION\fR.
+selected by \fBBINPKG_COMPRESS\fR.
.br
Defaults to "".
.br
.I Example:
.nf
# Set it to use compression level 9:
-BINPKG_COMPRESSION_ARGS="-9"
+BINPKG_COMPRESS_FLAGS="-9"
.fi
.TP
.B CBUILD
diff --git a/pym/portage/package/ebuild/_config/special_env_vars.py
b/pym/portage/package/ebuild/_config/special_env_vars.py
index f9b29af93..13da64008 100644
--- a/pym/portage/package/ebuild/_config/special_env_vars.py
+++ b/pym/portage/package/ebuild/_config/special_env_vars.py
@@ -147,6 +147,7 @@ environ_filter += [
environ_filter += [
"ACCEPT_CHOSTS", "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES",
"ACCEPT_RESTRICT", "AUTOCLEAN",
+ "BINPKG_COMPRESS", "BINPKG_COMPRESS_FLAGS",
"CLEAN_DELAY", "COLLISION_IGNORE",
"CONFIG_PROTECT", "CONFIG_PROTECT_MASK",
"DCO_SIGNED_OFF_BY",
diff --git a/pym/portage/package/ebuild/config.py
b/pym/portage/package/ebuild/config.py
index 3d232e0be..6691024f2 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -1172,12 +1172,12 @@ class config(object):
writemsg(_("!!! See
https://bugs.pypy.org/issue833 for details.\n"),
noiselevel=-1)
- binpkg_compression = self.get("BINPKG_COMPRESSION")
+ binpkg_compression = self.get("BINPKG_COMPRESS")
if binpkg_compression:
try:
compression = _compressors[binpkg_compression]
except KeyError as e:
- writemsg("!!! BINPKG_COMPRESSION contains
invalid or "
+ writemsg("!!! BINPKG_COMPRESS contains invalid
or "
"unsupported compression method: %s" %
e.args[0],
noiselevel=-1)
else:
@@ -1186,14 +1186,14 @@ class config(object):
portage.util.varexpand(compression["compress"],
mydict=self))[0]
except IndexError as e:
- writemsg("!!! BINPKG_COMPRESSION
contains invalid or "
+ writemsg("!!! BINPKG_COMPRESS contains
invalid or "
"unsupported compression
method: %s" %
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
commit: dac5089eb7908e9fd643f46c913515082077281e
Author: Zac Medico gentoo org>
AuthorDate: Fri May 5 09:07:38 2017 +
Commit: Zac Medico gentoo org>
CommitDate: Fri May 5 18:32:45 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dac5089e
Eventloop: fix deadlock involving idle_add/call_soon (bug 617550)
Guarantee that newly added idle_add/call_soon callbacks have an
opportunity to execute before the event loop decides to wait on
self._thread_condition without a timeout. This fixes a case where
the event loop would wait on self._thread_condition indefinitely,
even though a callback scheduled by the AsynchronousTask._async_wait
method needed to be executed first.
X-Gentoo-bug: 617550
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=617550
Acked-by: Brian Dolbec gentoo.org>
pym/portage/util/_eventloop/EventLoop.py | 18 --
1 file changed, 16 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 712838e3d..cd154005f 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -108,6 +108,15 @@ class EventLoop(object):
self._poll_event_handler_ids = {}
# Increment id for each new handler.
self._event_handler_id = 0
+ # New call_soon callbacks must have an opportunity to
+ # execute before it's safe to wait on self._thread_condition
+ # without a timeout, since delaying its execution indefinitely
+ # could lead to a deadlock. The following attribute stores the
+ # event handler id of the most recently added call_soon
callback.
+ # If this attribute has changed since the last time that the
+ # call_soon callbacks have been called, then it's not safe to
+ # wait on self._thread_condition without a timeout.
+ self._call_soon_id = 0
# Use OrderedDict in order to emulate the FIFO queue behavior
# of the AbstractEventLoop.call_soon method.
self._idle_callbacks = OrderedDict()
@@ -250,10 +259,15 @@ class EventLoop(object):
if not event_handlers:
with self._thread_condition:
+ prev_call_soon_id = self._call_soon_id
if self._run_timeouts():
events_handled += 1
timeouts_checked = True
- if not event_handlers and not events_handled
and may_block:
+
+ call_soon = prev_call_soon_id !=
self._call_soon_id
+
+ if (not call_soon and not event_handlers
+ and not events_handled and may_block):
# Block so that we don't waste cpu time
by looping too
# quickly. This makes EventLoop useful
for code that needs
# to wait for timeout callbacks
regardless of whether or
@@ -457,7 +471,7 @@ class EventLoop(object):
@return: an integer ID
"""
with self._thread_condition:
- source_id = self._new_source_id()
+ source_id = self._call_soon_id = self._new_source_id()
self._idle_callbacks[source_id] =
self._idle_callback_class(
args=args, callback=callback,
source_id=source_id)
self._thread_condition.notify()
[gentoo-commits] proj/portage:master commit in: pym/portage/util/futures/
commit: d90ff7c046b39de82558655375ea104cfa19b176 Author: Zac Medico gentoo org> AuthorDate: Sun Mar 26 20:08:54 2017 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 26 20:12:04 2017 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d90ff7c0 _EventLoopFuture: reduce indent of class body (whitespace only) pym/portage/util/futures/futures.py | 278 ++-- 1 file changed, 139 insertions(+), 139 deletions(-) diff --git a/pym/portage/util/futures/futures.py b/pym/portage/util/futures/futures.py index dd913a1e3..dcf593c01 100644 --- a/pym/portage/util/futures/futures.py +++ b/pym/portage/util/futures/futures.py @@ -42,148 +42,148 @@ _CANCELLED = 'CANCELLED' _FINISHED = 'FINISHED' class _EventLoopFuture(object): + """ + This class provides (a subset of) the asyncio.Future interface, for + use with the EventLoop class, because EventLoop is currently + missing some of the asyncio.AbstractEventLoop methods that + asyncio.Future requires. + """ + + # Class variables serving as defaults for instance variables. + _state = _PENDING + _result = None + _exception = None + _loop = None + + def __init__(self, loop=None): + """Initialize the future. + + The optional loop argument allows explicitly setting the event + loop object used by the future. If it's not provided, the future uses + the default event loop. """ - This class provides (a subset of) the asyncio.Future interface, for - use with the EventLoop class, because EventLoop is currently - missing some of the asyncio.AbstractEventLoop methods that - asyncio.Future requires. + if loop is None: + self._loop = global_event_loop() + else: + self._loop = loop + self._callbacks = [] + + def cancel(self): + """Cancel the future and schedule callbacks. + + If the future is already done or cancelled, return False. Otherwise, + change the future's state to cancelled, schedule the callbacks and + return True. """ + if self._state != _PENDING: + return False + self._state = _CANCELLED + self._schedule_callbacks() + return True - # Class variables serving as defaults for instance variables. - _state = _PENDING - _result = None - _exception = None - _loop = None - - def __init__(self, loop=None): - """Initialize the future. - - The optional loop argument allows explicitly setting the event - loop object used by the future. If it's not provided, the future uses - the default event loop. - """ - if loop is None: - self._loop = global_event_loop() - else: - self._loop = loop - self._callbacks = [] - - def cancel(self): - """Cancel the future and schedule callbacks. - - If the future is already done or cancelled, return False. Otherwise, - change the future's state to cancelled, schedule the callbacks and - return True. - """ - if self._state != _PENDING: - return False - self._state = _CANCELLED - self._schedule_callbacks() - return True - - def _schedule_callbacks(self): - """Internal: Ask the event loop to call all callbacks. - - The callbacks are scheduled to be called as soon as possible. Also - clears the callback list. - """ - callbacks = self._callbacks[:] - if not callbacks: - return - - self._callbacks[:] = [] - for callback in callbacks: - self._loop.call_soon(callback, self) - - def cancelled(self): - """Return True if the future was cancelled.""" - return self._state == _CANCELLED - - def done(self): - """Return True if the future is done. - - Done means either that a result / exception are available, or that the - future was cancelled. - """ - return self._state !=
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/, pym/portage/util/_async/, ...
commit: 4b12ed04ec6b99f5a948e0eea5778a4fac502740 Author: Zac Medico gentoo org> AuthorDate: Sun Mar 26 00:45:52 2017 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 26 20:05:46 2017 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4b12ed04 Future: implement add_done_callback for asyncio compat (bug 591760) Implement the add_done_callback and remove_done_callback methods, since they are required in order to make further progress toward asyncio compatibility. Also implement the AbstractEventLoop create_future method for the EventLoop class, so that it returns an instance of _EventLoopFuture. EventLoop currently does not implement some of the asyncio.AbstractEventLoop methods that asyncio.Future requires for its add_done_callback implementation, and the create_future method conveniently solves this problem. X-Gentoo-bug: 591760 X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=591760 Acked-by: Brian Dolbec gentoo.org> pym/portage/tests/ebuild/test_ipc_daemon.py| 3 +- .../tests/util/eventloop/test_call_soon_fifo.py| 6 +- pym/portage/tests/util/futures/__init__.py | 0 pym/portage/tests/util/futures/__test__.py | 0 .../tests/util/futures/test_done_callback.py | 35 + pym/portage/util/_async/SchedulerInterface.py | 3 +- pym/portage/util/_eventloop/EventLoop.py | 14 pym/portage/util/futures/futures.py| 82 -- 8 files changed, 132 insertions(+), 11 deletions(-) diff --git a/pym/portage/tests/ebuild/test_ipc_daemon.py b/pym/portage/tests/ebuild/test_ipc_daemon.py index 68f139aa4..fc7916541 100644 --- a/pym/portage/tests/ebuild/test_ipc_daemon.py +++ b/pym/portage/tests/ebuild/test_ipc_daemon.py @@ -16,7 +16,6 @@ from portage.util import ensure_dirs from portage.util._async.ForkProcess import ForkProcess from portage.util._async.TaskScheduler import TaskScheduler from portage.util._eventloop.global_event_loop import global_event_loop -from portage.util.futures.futures import Future from _emerge.SpawnProcess import SpawnProcess from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EbuildIpcDaemon import EbuildIpcDaemon @@ -150,7 +149,7 @@ class IpcDaemonTestCase(TestCase): self._run_done.set_result(True) def _run(self, event_loop, task_scheduler, timeout): - self._run_done = Future() + self._run_done = event_loop.create_future() timeout_id = event_loop.timeout_add(timeout, self._timeout_callback, task_scheduler) task_scheduler.addExitListener(self._exit_callback) diff --git a/pym/portage/tests/util/eventloop/test_call_soon_fifo.py b/pym/portage/tests/util/eventloop/test_call_soon_fifo.py index 5ecc13f43..f970c67a1 100644 --- a/pym/portage/tests/util/eventloop/test_call_soon_fifo.py +++ b/pym/portage/tests/util/eventloop/test_call_soon_fifo.py @@ -7,22 +7,22 @@ import random from portage import os from portage.tests import TestCase from portage.util._eventloop.global_event_loop import global_event_loop -from portage.util.futures.futures import Future + class CallSoonFifoTestCase(TestCase): def testCallSoonFifo(self): + event_loop = global_event_loop() inputs = [random.random() for index in range(10)] outputs = [] - finished = Future() + finished = event_loop.create_future() def add_output(value): outputs.append(value) if len(outputs) == len(inputs): finished.set_result(True) - event_loop = global_event_loop() for value in inputs: event_loop.call_soon(functools.partial(add_output, value)) diff --git a/pym/portage/tests/util/futures/__init__.py b/pym/portage/tests/util/futures/__init__.py new file mode 100644 index 0..e69de29bb diff --git a/pym/portage/tests/util/futures/__test__.py b/pym/portage/tests/util/futures/__test__.py new file mode 100644 index 0..e69de29bb diff --git a/pym/portage/tests/util/futures/test_done_callback.py b/pym/portage/tests/util/futures/test_done_callback.py new file mode 100644 index 0..76b727b09 --- /dev/null +++ b/pym/portage/tests/util/futures/test_done_callback.py @@ -0,0 +1,35 @@ +# Copyright 2017 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.util._eventloop.global_event_loop import global_event_loop + + +class FutureDoneCallbackTestCase(TestCase): + + def testFutureDoneCallback(self): + + event_loop = global_event_loop() + + def done_callback(finished): + done_callback_called.set_result(True) + + done_callback_called = event_loop.create_future() +
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/, pym/portage/tests/util/eventloop/, ...
commit: 04b1012594bfad1be719547e2c88a2dcf1051dc1
Author: Zac Medico gentoo org>
AuthorDate: Tue Mar 21 06:54:47 2017 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Mar 24 20:32:11 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=04b10125
EventLoop: implement call_soon for asyncio compat (bug 591760)
Since asyncio.AbstractEventLoop has no equivalent to the idle
callbacks implemented by the EventLoop.idle_add method, it is
necessary to implement the AbstractEventLoop.call_soon and
call_soon_threadsafe methods, so that idle_add usage can
eventually be eliminated.
X-Gentoo-bug: 591760
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=591760
Acked-by: Brian Dolbec gentoo.org>
.../tests/util/eventloop/test_call_soon_fifo.py| 30 ++
pym/portage/util/_async/SchedulerInterface.py | 5 +-
pym/portage/util/_eventloop/EventLoop.py | 67 +-
3 files changed, 99 insertions(+), 3 deletions(-)
diff --git a/pym/portage/tests/util/eventloop/test_call_soon_fifo.py
b/pym/portage/tests/util/eventloop/test_call_soon_fifo.py
new file mode 100644
index 0..5ecc13f43
--- /dev/null
+++ b/pym/portage/tests/util/eventloop/test_call_soon_fifo.py
@@ -0,0 +1,30 @@
+# Copyright 2017 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import functools
+import random
+
+from portage import os
+from portage.tests import TestCase
+from portage.util._eventloop.global_event_loop import global_event_loop
+from portage.util.futures.futures import Future
+
+class CallSoonFifoTestCase(TestCase):
+
+ def testCallSoonFifo(self):
+
+ inputs = [random.random() for index in range(10)]
+ outputs = []
+ finished = Future()
+
+ def add_output(value):
+ outputs.append(value)
+ if len(outputs) == len(inputs):
+ finished.set_result(True)
+
+ event_loop = global_event_loop()
+ for value in inputs:
+ event_loop.call_soon(functools.partial(add_output,
value))
+
+ event_loop.run_until_complete(finished)
+ self.assertEqual(inputs, outputs)
diff --git a/pym/portage/util/_async/SchedulerInterface.py
b/pym/portage/util/_async/SchedulerInterface.py
index 2ab668ee4..6028fd90d 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -13,8 +13,9 @@ class SchedulerInterface(SlotObject):
_event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN",
"IO_NVAL", "IO_OUT", "IO_PRI",
- "child_watch_add", "idle_add", "io_add_watch",
- "iteration", "source_remove", "timeout_add")
+ "call_soon", "call_soon_threadsafe", "child_watch_add",
+ "idle_add", "io_add_watch", "iteration", "run_until_complete",
+ "source_remove", "timeout_add")
__slots__ = _event_loop_attrs + ("_event_loop", "_is_background")
diff --git a/pym/portage/util/_eventloop/EventLoop.py
b/pym/portage/util/_eventloop/EventLoop.py
index 8f13de377..308157bea 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -22,6 +22,7 @@ try:
except ImportError:
import dummy_threading as threading
+from portage import OrderedDict
from portage.util import writemsg_level
from ..SlotObject import SlotObject
from .PollConstants import PollConstants
@@ -54,6 +55,38 @@ class EventLoop(object):
__slots__ = ("args", "function", "calling", "interval",
"source_id",
"timestamp")
+ class _handle(object):
+ """
+ A callback wrapper object, compatible with asyncio.Handle.
+ """
+ __slots__ = ("_callback_id", "_loop")
+
+ def __init__(self, callback_id, loop):
+ self._callback_id = callback_id
+ self._loop = loop
+
+ def cancel(self):
+ """
+ Cancel the call. If the callback is already canceled or
executed,
+ this method has no effect.
+ """
+ self._loop.source_remove(self._callback_id)
+
+ class _call_soon_callback(object):
+ """
+ Wraps a call_soon callback, and always returns False, since
these
+ callbacks are only supposed to run once.
+ """
+ __slots__ = ("_args", "_callback")
+
+ def __init__(self, callback, args):
+ self._callback = callback
+ self._args = args
+
+ def __call__(self):
+ self._callback(*self._args)
+ return False
+
def __init__(self, main=True):
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, pym/_emerge/
commit: 86400e9f864e86f8f677ccda9ce4103d6d02ef87 Author: Zac Medico gentoo org> AuthorDate: Tue Mar 21 06:56:55 2017 + Commit: Zac Medico gentoo org> CommitDate: Fri Mar 24 20:32:25 2017 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=86400e9f PollScheduler: terminate via call_soon for asyncio compat Use call_soon to schedule the _termination_check callback when needed. The previous idle_add usage was relatively inefficient, because it scheduled the _termination_check callback to be called in every iteration of the event loop. Add a _cleanup method to handle cleanup of callbacks registered with the global event loop. Since the terminate method is thread safe and it interacts with self._term_callback_handle, use this variable only while holding a lock. pym/_emerge/PollScheduler.py | 57 +++ pym/_emerge/Scheduler.py | 7 ++-- pym/portage/util/_async/AsyncScheduler.py | 16 - 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/pym/_emerge/PollScheduler.py b/pym/_emerge/PollScheduler.py index b118ac157..569879b36 100644 --- a/pym/_emerge/PollScheduler.py +++ b/pym/_emerge/PollScheduler.py @@ -25,8 +25,10 @@ class PollScheduler(object): a non-main thread) @type main: bool """ + self._term_rlock = threading.RLock() self._terminated = threading.Event() self._terminated_tasks = False + self._term_check_handle = None self._max_jobs = 1 self._max_load = None self._scheduling = False @@ -44,6 +46,21 @@ class PollScheduler(object): def _is_background(self): return self._background + def _cleanup(self): + """ + Cleanup any callbacks that have been registered with the global + event loop. + """ + # The self._term_check_handle attribute requires locking + # since it's modified by the thread safe terminate method. + with self._term_rlock: + if self._term_check_handle not in (None, False): + self._term_check_handle.cancel() + # This prevents the terminate method from scheduling + # any more callbacks (since _cleanup must eliminate all + # callbacks in order to ensure complete cleanup). + self._term_check_handle = False + def terminate(self): """ Schedules asynchronous, graceful termination of the scheduler @@ -51,26 +68,36 @@ class PollScheduler(object): This method is thread-safe (and safe for signal handlers). """ - self._terminated.set() + with self._term_rlock: + if self._term_check_handle is None: + self._terminated.set() + self._term_check_handle = self._event_loop.call_soon_threadsafe( + self._termination_check, True) - def _termination_check(self): + def _termination_check(self, retry=False): """ Calls _terminate_tasks() if appropriate. It's guaranteed not to - call it while _schedule_tasks() is being called. The check should - be executed for each iteration of the event loop, for response to - termination signals at the earliest opportunity. It always returns - True, for continuous scheduling via idle_add. + call it while _schedule_tasks() is being called. This method must + only be called via the event loop thread. + + @param retry: If True then reschedule if scheduling state prevents + immediate termination. + @type retry: bool """ - if not self._scheduling and \ - self._terminated.is_set() and \ + if self._terminated.is_set() and \ not self._terminated_tasks: - self._scheduling = True - try: - self._terminated_tasks = True - self._terminate_tasks() - finally: - self._scheduling = False - return True + if not self._scheduling: + self._scheduling = True + try: + self._terminated_tasks = True + self._terminate_tasks() + finally: + self._scheduling = False + +
[gentoo-commits] proj/portage:master commit in: /, pym/portage/util/file_copy/, src/, pym/portage/util/, ...
commit: 8ab5c8835931fd9ec098dbf4c5f416eb32e4a3a4 Author: Zac Medico gentoo org> AuthorDate: Thu Feb 2 03:14:53 2017 + Commit: Zac Medico gentoo org> CommitDate: Thu Mar 16 02:38:37 2017 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8ab5c883 movefile: support in-kernel file copying on Linux (bug 607868) Perform in-kernel file copying when possible, and also support reflinks and sparse files. If the optimized implementation fails at runtime, gracefully fallback to a plain read/write loop. Compile-time and run-time fallbacks are implemented, so that any incompatiblities will be handled gracefully. For example, if the code is compiled on a system that supports the copy_file_range syscall, but at run-time an older kernel that does not support this syscall is detected, it will be handled gracefully. There are similar fallbacks for lack of lseek SEEK_DATA and sendfile support. X-Gentoo-Bug: 607868 X-Gentoo-Bug-Url: https://bugs.gentoo.org/show_bug.cgi?id=607868 Acked-by: Brian Dolbec gentoo.org> pym/portage/tests/util/file_copy/__init__.py | 0 pym/portage/tests/util/file_copy/__test__.py | 0 pym/portage/tests/util/file_copy/test_copyfile.py | 71 pym/portage/util/file_copy/__init__.py| 36 ++ pym/portage/util/movefile.py | 5 +- setup.py | 9 + src/portage_util_file_copy_reflink_linux.c| 385 ++ 7 files changed, 504 insertions(+), 2 deletions(-) diff --git a/pym/portage/tests/util/file_copy/__init__.py b/pym/portage/tests/util/file_copy/__init__.py new file mode 100644 index 0..e69de29bb diff --git a/pym/portage/tests/util/file_copy/__test__.py b/pym/portage/tests/util/file_copy/__test__.py new file mode 100644 index 0..e69de29bb diff --git a/pym/portage/tests/util/file_copy/test_copyfile.py b/pym/portage/tests/util/file_copy/test_copyfile.py new file mode 100644 index 0..b900fdef0 --- /dev/null +++ b/pym/portage/tests/util/file_copy/test_copyfile.py @@ -0,0 +1,71 @@ +# Copyright 2017 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import shutil +import tempfile + +from portage import os +from portage.tests import TestCase +from portage.checksum import perform_md5 +from portage.util.file_copy import copyfile + + +class CopyFileTestCase(TestCase): + + def testCopyFile(self): + + tempdir = tempfile.mkdtemp() + try: + src_path = os.path.join(tempdir, 'src') + dest_path = os.path.join(tempdir, 'dest') + content = b'foo' + + with open(src_path, 'wb') as f: + f.write(content) + + copyfile(src_path, dest_path) + + self.assertEqual(perform_md5(src_path), perform_md5(dest_path)) + finally: + shutil.rmtree(tempdir) + + +class CopyFileSparseTestCase(TestCase): + + def testCopyFileSparse(self): + + tempdir = tempfile.mkdtemp() + try: + src_path = os.path.join(tempdir, 'src') + dest_path = os.path.join(tempdir, 'dest') + content = b'foo' + + # Use seek to create some sparse blocks. Don't make these + # files too big, in case the filesystem doesn't support + # sparse files. + with open(src_path, 'wb') as f: + f.write(content) + f.seek(2**17, 1) + f.write(content) + f.seek(2**18, 1) + f.write(content) + # Test that sparse blocks are handled correctly at + # the end of the file (involves seek and truncate). + f.seek(2**17, 1) + + copyfile(src_path, dest_path) + + self.assertEqual(perform_md5(src_path), perform_md5(dest_path)) + + # This last part of the test is expected to fail when sparse + # copy is not implemented, so set the todo flag in order + # to tolerate failures. + self.todo = True + + # If sparse blocks were preserved, then both files should + # consume the same number of blocks. + self.assertEqual( + os.stat(src_path).st_blocks, + os.stat(dest_path).st_blocks) + finally: + shutil.rmtree(tempdir) diff --git a/pym/portage/util/file_copy/__init__.py b/pym/portage/util/file_copy/__init__.py new file mode 100644 index
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: f4b0714010f1237280fd48dcb65989679917d20a Author: Zac Medico gentoo org> AuthorDate: Thu Feb 23 10:50:36 2017 + Commit: Zac Medico gentoo org> CommitDate: Thu Feb 23 15:59:43 2017 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f4b07140 grabfile_package: support -* in profile "packages" files (bug 610670) Support -* in order to make it easier to create profiles for minimal systems (especially those built entirely from binary packages). X-Gentoo-Bug: 610670 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=610670 Acked-by: Brian Dolbec gentoo.org> pym/portage/util/__init__.py | 9 - 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index c2c871ffb..45710ba72 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -478,13 +478,20 @@ def grabfile_package(myfilename, compatlevel=0, recursive=0, eapi = read_corresponding_eapi_file( myfilename, default=eapi_default) mybasename = os.path.basename(myfilename) + is_packages_file = mybasename == 'packages' atoms = [] for pkg, source_file in pkgs: pkg_orig = pkg # for packages and package.mask files if pkg[:1] == "-": + if is_packages_file and pkg == '-*': + if remember_source_file: + atoms.append((pkg, source_file)) + else: + atoms.append(pkg) + continue pkg = pkg[1:] - if pkg[:1] == '*' and mybasename == 'packages': + if pkg[:1] == '*' and is_packages_file: pkg = pkg[1:] try: pkg = Atom(pkg, allow_wildcard=allow_wildcard,
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/
commit: 855c8761f85323bbe570ae61e87c57dc8ea5ea0e Author: Zac Medico gentoo org> AuthorDate: Wed Feb 8 09:08:07 2017 + Commit: Zac Medico gentoo org> CommitDate: Wed Feb 8 09:10:49 2017 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=855c8761 PopenProcess: suppress ResourceWarning subprocess "still running" (bug 608594) Override the _set_returncode method to set the Popen.returncode attribute, in order to suppress Python 3.6 ResourceWarnings which erroneously report that the subprocess is still running. X-Gentoo-Bug: 608594 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=608594 pym/portage/util/_async/PopenProcess.py | 9 - 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/_async/PopenProcess.py b/pym/portage/util/_async/PopenProcess.py index 2fc56d295..4344b1c9d 100644 --- a/pym/portage/util/_async/PopenProcess.py +++ b/pym/portage/util/_async/PopenProcess.py @@ -1,4 +1,4 @@ -# Copyright 2012 Gentoo Foundation +# Copyright 2012-2017 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.SubProcess import SubProcess @@ -31,3 +31,10 @@ class PopenProcess(SubProcess): self._reg_id = None self._waitpid_cb(pid, condition) self.wait() + + def _set_returncode(self, wait_retval): + SubProcess._set_returncode(self, wait_retval) + if self.proc.returncode is None: + # Suppress warning messages like this: + # ResourceWarning: subprocess 1234 is still running + self.proc.returncode = self.returncode
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: b3401efea5820c525c57a1b52c38404054593e0a
Author: Zac Medico gentoo org>
AuthorDate: Sun Jan 22 18:24:07 2017 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Jan 22 18:24:47 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b3401efe
env-update: skip os.access call when ldconfig is None (bug 606832)
Since commit 1bc49bead14ddd31c94921fe9c3d1972f0737056, env_update
would raise the following exception with crossdev configurations
when ${CHOST}-ldconfig is not available:
TypeError: access: can't specify None for path argument
Fixes: 1bc49bead14d ("env-update: call ldconfig if found in EROOT")
X-Gentoo-Bug: 606832
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=606832
pym/portage/util/env_update.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/env_update.py b/pym/portage/util/env_update.py
index fde2f66..0321010 100644
--- a/pym/portage/util/env_update.py
+++ b/pym/portage/util/env_update.py
@@ -312,7 +312,9 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env,
else:
ldconfig = os.path.join(eroot, "sbin", "ldconfig")
- if not (os.access(ldconfig, os.X_OK) and os.path.isfile(ldconfig)):
+ if ldconfig is None:
+ pass
+ elif not (os.access(ldconfig, os.X_OK) and os.path.isfile(ldconfig)):
ldconfig = None
# Only run ldconfig as needed
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 1bc49bead14ddd31c94921fe9c3d1972f0737056
Author: Benda Xu gentoo org>
AuthorDate: Mon Jun 20 00:12:35 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Jan 7 22:47:32 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1bc49bea
env-update: call ldconfig if found in EROOT
prefix-standalone has a glibc installed by portage with ldconfig
under EROOT/sbin. It should be called during env-update.
For prefix-rpath, host glibc is used and env-update should not
care about ldconfig. In this case, no ldconfig is in EROOT/sbin
and ldconfig is skipped.
X-Gentoo-Bug: 532100
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532100
pym/portage/util/env_update.py | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/env_update.py b/pym/portage/util/env_update.py
index c0a93a8..fde2f66 100644
--- a/pym/portage/util/env_update.py
+++ b/pym/portage/util/env_update.py
@@ -306,13 +306,17 @@ def _env_update(makelinks, target_root, prev_mtimes,
contents, env,
if not libdir_contents_changed:
makelinks = False
- ldconfig = "/sbin/ldconfig"
if "CHOST" in settings and "CBUILD" in settings and \
settings["CHOST"] != settings["CBUILD"]:
ldconfig = find_binary("%s-ldconfig" % settings["CHOST"])
+ else:
+ ldconfig = os.path.join(eroot, "sbin", "ldconfig")
+
+ if not (os.access(ldconfig, os.X_OK) and os.path.isfile(ldconfig)):
+ ldconfig = None
# Only run ldconfig as needed
- if makelinks and ldconfig and not eprefix:
+ if makelinks and ldconfig:
# ldconfig has very different behaviour between FreeBSD and
Linux
if ostype == "Linux" or ostype.lower().endswith("gnu"):
# We can't update links if we haven't cleaned other
versions first, as
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
commit: cbaee3c3b28f52947c142da8df24d6b0817962f9
Author: Zac Medico gentoo org>
AuthorDate: Tue Dec 27 04:17:01 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Dec 27 21:25:06 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cbaee3c3
LinkageMapELF: compute multilib category for preserved libs (bug 598080)
When support for parsing ELF headers in order to compute multilib
category was added in commit f1c1b8a77eebf7713b32e5f9945690f60f4f46de,
the LinkageMapELF class was not updated to do the same for preserved
libraries. This has resulted in incorrect preserve-libs handling
as reported in bug 598080, for ABIs including x32 where the
_approx_multilib_categories mapping is insufficient. This patch fixes
LinkageMapELF to compute the multilib category for each preserved
library, in the same way as the _post_src_install_soname_symlinks
function, so that the LinkageMapELF.findConsumers method will operate
correctly with preserved libraries of all ABIs.
Fixes: f1c1b8a77eeb ("Generate soname dependency metadata (bug 282639)")
X-Gentoo-bug: 598080
X-Gentoo-bug-url: https://bugs.gentoo.org/598080
Acked-by: Brian Dolbec gentoo.org>
pym/portage/util/_dyn_libs/LinkageMapELF.py | 33 +++--
1 file changed, 26 insertions(+), 7 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py
b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 0b09fe5..a063621 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -4,6 +4,7 @@
import errno
import logging
import subprocess
+import sys
import portage
from portage import _encodings
@@ -12,6 +13,7 @@ from portage import _unicode_decode
from portage import _unicode_encode
from portage.cache.mappings import slot_dict_class
from portage.const import EPREFIX
+from portage.dep.soname.multilib_category import compute_multilib_category
from portage.exception import CommandNotFound, InvalidData
from portage.localization import _
from portage.util import getlibpaths
@@ -20,6 +22,12 @@ from portage.util import normalize_path
from portage.util import varexpand
from portage.util import writemsg_level
from portage.util._dyn_libs.NeededEntry import NeededEntry
+from portage.util.elf.header import ELFHeader
+
+if sys.hexversion >= 0x300:
+ _unicode = str
+else:
+ _unicode = unicode
# Map ELF e_machine values from NEEDED.ELF.2 to approximate multilib
# categories. This approximation will produce incorrect results on x32
@@ -283,15 +291,26 @@ class LinkageMapELF(object):
l = l[3:].rstrip("\n")
if not l:
continue
- fields = l.split(";")
- if len(fields) < 5:
- writemsg_level(_("\nWrong
number of fields " \
- "returned from scanelf:
%s\n\n") % (l,),
+ try:
+ entry =
NeededEntry.parse("scanelf", l)
+ except InvalidData as e:
+ writemsg_level("\n%s\n\n" %
(e,),
level=logging.ERROR,
noiselevel=-1)
continue
- fields[1] = fields[1][root_len:]
- owner = plibs.pop(fields[1], None)
- lines.append((owner, "scanelf",
";".join(fields)))
+ try:
+ with
open(_unicode_encode(entry.filename,
+
encoding=_encodings['fs'],
+ errors='strict'), 'rb')
as f:
+ elf_header =
ELFHeader.read(f)
+ except EnvironmentError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ # File removed concurrently.
+ continue
+ entry.multilib_category =
compute_multilib_category(elf_header)
+ entry.filename =
entry.filename[root_len:]
+ owner = plibs.pop(entry.filename, None)
+ lines.append((owner, "scanelf",
_unicode(entry)))
proc.wait()
proc.stdout.close()
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 2593cb2c5d56e373c6c6e4f4663f25241e0e79b7
Author: Mikhail Pukhlikov gentoo org>
AuthorDate: Fri Sep 16 08:59:53 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Sep 18 22:13:04 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2593cb2c
writeable_check: handle invalid entries in /proc/self/mountinfo
That fixes Gentoo installation won WLS
Source:
https://github.com/Microsoft/BashOnWindows/issues/992#issuecomment-244460439
Can reproduce on two machines
pym/portage/util/writeable_check.py | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/writeable_check.py
b/pym/portage/util/writeable_check.py
index ac6c039..b698ea1 100644
--- a/pym/portage/util/writeable_check.py
+++ b/pym/portage/util/writeable_check.py
@@ -58,7 +58,13 @@ def linux_ro_checker(dir_list):
# to the left of the ' - ', after the attr's,
so split it there
mount = line.split(' - ', 1)
_dir, attr1 = mount[0].split()[4:6]
- attr2 = mount[1].split()[2]
+ # check for situation with invalid entries for
/home and /root in /proc/self/mountinfo
+ # root path is missing sometimes on WSL
+ # for example: 16 1 0:16 / /root rw,noatime -
lxfs rw
+ try:
+ attr2 = mount[1].split()[2]
+ except IndexError:
+ attr2 = mount[1].split()[1]
if attr1.startswith('ro') or
attr2.startswith('ro'):
ro_filesystems.add(_dir)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 9b31cd9f2823d742ce3431e480561244f00f1744
Author: Zac Medico gentoo org>
AuthorDate: Sun Sep 18 22:10:54 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Sep 18 22:13:45 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9b31cd9f
writeable_check: warn about invalid mountinfo lines (bug 594284)
X-Gentoo-bug: 594284
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=594284
pym/portage/util/writeable_check.py | 12 +++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/writeable_check.py
b/pym/portage/util/writeable_check.py
index 26fe199..e5b14c0 100644
--- a/pym/portage/util/writeable_check.py
+++ b/pym/portage/util/writeable_check.py
@@ -44,6 +44,7 @@ def linux_ro_checker(dir_list):
read-only, may be empty.
"""
ro_filesystems = set()
+ invalids = []
try:
with io.open("/proc/self/mountinfo", mode='r',
@@ -61,6 +62,7 @@ def linux_ro_checker(dir_list):
_dir, attr1 = mount[0].split()[4:6]
except ValueError:
# If it raises ValueError we can simply
ignore the line.
+ invalids.append(line)
continue
# check for situation with invalid entries for
/home and /root in /proc/self/mountinfo
# root path is missing sometimes on WSL
@@ -72,7 +74,11 @@ def linux_ro_checker(dir_list):
try:
attr2 =
mount[1].split()[1]
except IndexError:
- attr2 = mount[1]
+ invalids.append(line)
+ continue
+ else:
+ invalids.append(line)
+ continue
if attr1.startswith('ro') or
attr2.startswith('ro'):
ro_filesystems.add(_dir)
@@ -83,6 +89,10 @@ def linux_ro_checker(dir_list):
level=logging.WARNING, noiselevel=-1)
return []
+ for line in invalids:
+ writemsg_level(_("!!! /proc/self/mountinfo contains
unrecognized line: %s\n")
+ % line.rstrip(), level=logging.WARNING, noiselevel=-1)
+
ro_devs = {}
for x in ro_filesystems:
try:
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: a000c492b9f12a43b75c2613323e29a472732259
Author: Mikhail Pukhlikov gentoo org>
AuthorDate: Fri Sep 16 10:47:22 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Sep 18 22:13:18 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a000c492
writeable_check: add additional checks to mountinfo parsing function
pym/portage/util/writeable_check.py | 18 +-
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/pym/portage/util/writeable_check.py
b/pym/portage/util/writeable_check.py
index b698ea1..26fe199 100644
--- a/pym/portage/util/writeable_check.py
+++ b/pym/portage/util/writeable_check.py
@@ -57,14 +57,22 @@ def linux_ro_checker(dir_list):
# there can be a variable number of fields
# to the left of the ' - ', after the attr's,
so split it there
mount = line.split(' - ', 1)
- _dir, attr1 = mount[0].split()[4:6]
+ try:
+ _dir, attr1 = mount[0].split()[4:6]
+ except ValueError:
+ # If it raises ValueError we can simply
ignore the line.
+ continue
# check for situation with invalid entries for
/home and /root in /proc/self/mountinfo
# root path is missing sometimes on WSL
# for example: 16 1 0:16 / /root rw,noatime -
lxfs rw
- try:
- attr2 = mount[1].split()[2]
- except IndexError:
- attr2 = mount[1].split()[1]
+ if len(mount) > 1:
+ try:
+ attr2 = mount[1].split()[2]
+ except IndexError:
+ try:
+ attr2 =
mount[1].split()[1]
+ except IndexError:
+ attr2 = mount[1]
if attr1.startswith('ro') or
attr2.startswith('ro'):
ro_filesystems.add(_dir)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 3f8f3263a316159359d4137d77c7be2c80fc9d57
Author: Anthony G. Basile gentoo org>
AuthorDate: Wed May 25 12:48:32 2016 +
Commit: Brian Dolbec gentoo org>
CommitDate: Wed Sep 14 21:53:47 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3f8f3263
pym/portage/util/locale.py: fix decoding for python2 plus some locales
When using python2 with some locales, like turkish, chr() is passed values not
in
range(128) which cannot be decoded as ASCII, thus throwing a UnicodeDecodeError
exception. We use _unicode_decode() from portage.util to address this.
Signed-off-by: Anthony G. Basile gentoo.org>
pym/portage/util/locale.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
index 2a15ea1..093eb86 100644
--- a/pym/portage/util/locale.py
+++ b/pym/portage/util/locale.py
@@ -15,7 +15,7 @@ import textwrap
import traceback
import portage
-from portage.util import writemsg_level
+from portage.util import _unicode_decode, writemsg_level
from portage.util._ctypes import find_library, LoadLibrary
@@ -62,7 +62,7 @@ def _check_locale(silent):
"as LC_CTYPE in make.conf.")
msg = [l for l in textwrap.wrap(msg, 70)]
msg.append("")
- chars = lambda l: ''.join(chr(x) for x in l)
+ chars = lambda l: ''.join(_unicode_decode(chr(x)) for x in l)
if uc != ruc:
msg.extend([
" %s -> %s" % (chars(lc), chars(ruc)),
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
commit: 54d3676d4dc90444895c99c0b6c29d6be6a25b77
Author: Benda Xu gentoo org>
AuthorDate: Mon Jun 6 16:00:59 2016 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jun 6 16:00:59 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=54d3676d
LinkageMapELF: Account for EPREFIX in scanelf path (bug 583754)
X-Gentoo-Bug: 583754
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=583754
pym/portage/util/_dyn_libs/LinkageMapELF.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py
b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 63e2213..0b09fe5 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2013 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -11,6 +11,7 @@ from portage import _os_merge
from portage import _unicode_decode
from portage import _unicode_encode
from portage.cache.mappings import slot_dict_class
+from portage.const import EPREFIX
from portage.exception import CommandNotFound, InvalidData
from portage.localization import _
from portage.util import getlibpaths
@@ -259,7 +260,7 @@ class LinkageMapELF(object):
continue
plibs.update((x, cpv) for x in items)
if plibs:
- args = ["/usr/bin/scanelf", "-qF", "%a;%F;%S;%r;%n"]
+ args = [os.path.join(EPREFIX or "/",
"usr/bin/scanelf"), "-qF", "%a;%F;%S;%r;%n"]
args.extend(os.path.join(root, x.lstrip("." + os.sep)) \
for x in plibs)
try:
[gentoo-commits] proj/portage:master commit in: pym/portage/util/, pym/portage/repository/, pym/portage/_sets/
commit: 7a8392286ce3c5b2d33abe2de98922997307c182 Author: Michał Górny gentoo org> AuthorDate: Sun May 22 07:48:10 2016 + Commit: Michał Górny gentoo org> CommitDate: Tue May 24 06:27:58 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a839228 portage.util.configparser: Provide common code to handle cp imports Provide a common code unit to handle portable *ConfigParser imports for all supported Python versions. Reviewed-by: Zac Medico gentoo.org> pym/portage/_sets/__init__.py | 10 ++ pym/portage/repository/config.py | 9 + pym/portage/util/_desktop_entry.py | 7 ++- pym/portage/util/configparser.py | 22 ++ 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/pym/portage/_sets/__init__.py b/pym/portage/_sets/__init__.py index 3203521..ec42f7c 100644 --- a/pym/portage/_sets/__init__.py +++ b/pym/portage/_sets/__init__.py @@ -9,14 +9,6 @@ __all__ = ["SETPREFIX", "get_boolean", "SetConfigError", import io import logging import sys -try: - from configparser import NoOptionError, ParsingError - if sys.hexversion >= 0x302: - from configparser import ConfigParser as SafeConfigParser - else: - from configparser import SafeConfigParser -except ImportError: - from ConfigParser import SafeConfigParser, NoOptionError, ParsingError import portage from portage import os from portage import load_mod @@ -29,6 +21,8 @@ from portage.const import _ENABLE_SET_CONFIG from portage.exception import PackageSetNotFound from portage.localization import _ from portage.util import writemsg_level +from portage.util.configparser import (SafeConfigParser, + NoOptionError, ParsingError) SETPREFIX = "@" diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index 00319fe..9039886 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -9,14 +9,6 @@ import warnings import sys import re -try: - from configparser import Error as ConfigParserError - if sys.hexversion >= 0x302: - from configparser import ConfigParser as SafeConfigParser - else: - from configparser import SafeConfigParser -except ImportError: - from ConfigParser import SafeConfigParser, Error as ConfigParserError import portage from portage import eclass_cache, os from portage.const import (MANIFEST2_HASH_FUNCTIONS, MANIFEST2_REQUIRED_HASH, @@ -25,6 +17,7 @@ from portage.eapi import eapi_allows_directories_on_profile_level_and_repository from portage.env.loaders import KeyValuePairFileLoader from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split, stack_lists, writemsg, writemsg_level, _recursive_file_list) +from portage.util.configparser import SafeConfigParser, ConfigParserError from portage.util._path import isdir_raise_eaccess from portage.util.path import first_existing from portage.localization import _ diff --git a/pym/portage/util/_desktop_entry.py b/pym/portage/util/_desktop_entry.py index 0b49547..95a015e 100644 --- a/pym/portage/util/_desktop_entry.py +++ b/pym/portage/util/_desktop_entry.py @@ -6,14 +6,11 @@ import re import subprocess import sys -try: - from configparser import Error as ConfigParserError, RawConfigParser -except ImportError: - from ConfigParser import Error as ConfigParserError, RawConfigParser - import portage from portage import _encodings, _unicode_encode, _unicode_decode from portage.util import writemsg +from portage.util.configparser import ConfigParserError, RawConfigParser + def parse_desktop_entry(path): """ diff --git a/pym/portage/util/configparser.py b/pym/portage/util/configparser.py new file mode 100644 index 000..d305052 --- /dev/null +++ b/pym/portage/util/configparser.py @@ -0,0 +1,22 @@ +# Copyright 2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +__all__ = ['ConfigParserError', 'NoOptionError', 'ParsingError', + 'RawConfigParser', 'SafeConfigParser'] + +# the following scary compatibility thing provides two classes: +# - SafeConfigParser that provides safe interpolation for values, +# - RawConfigParser that provides no interpolation for values. + +import sys + +try: + from configparser import (Error as ConfigParserError, + NoOptionError, ParsingError, RawConfigParser) + if sys.hexversion >= 0x302: + from configparser import ConfigParser as SafeConfigParser + else: + from configparser import SafeConfigParser +except ImportError: + from ConfigParser import (Error as ConfigParserError, + NoOptionError, ParsingError, RawConfigParser, SafeConfigParser)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 59f2371c58ea6942ad441d8238600c21cda1c014 Author: Zac Medico gentoo org> AuthorDate: Sun Nov 22 20:51:10 2015 + Commit: Zac Medico gentoo org> CommitDate: Sun Nov 22 20:51:27 2015 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=59f2371c portage.util.locale: fix python2.7 setlocale ValueError (bug 566372) The locale.setlocale function raises ValueError with python2.7 if it is given a unicode argument. X-Gentoo-Bug: 566372 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=566372 pym/portage/util/locale.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py index 05b3979..2a15ea1 100644 --- a/pym/portage/util/locale.py +++ b/pym/portage/util/locale.py @@ -14,6 +14,7 @@ import os import textwrap import traceback +import portage from portage.util import writemsg_level from portage.util._ctypes import find_library, LoadLibrary @@ -102,7 +103,8 @@ def check_locale(silent=False, env=None): try: if env is not None: try: - locale.setlocale(locale.LC_CTYPE, mylocale) + locale.setlocale(locale.LC_CTYPE, + portage._native_string(mylocale)) except locale.Error: os._exit(2)
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: ab7efe823ace977d8b46f324bdcbf44dcb7ca6d9
Author: Zac Medico gentoo org>
AuthorDate: Sat Nov 21 18:11:17 2015 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Nov 21 18:11:43 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ab7efe82
portage.util.locale: enable absolute_import (bug 566372)
Since commit e5ba8d096e56495a9b516cea818d48e104d62bca,
absolute_import is required for python2 compatibility.
Fixes: e5ba8d096e56 ("EAPI 6: Enforce posixish LC_CTYPE")
X-Gentoo-Bug: 566372
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=566372
pym/portage/util/locale.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
index c3332ea..05b3979 100644
--- a/pym/portage/util/locale.py
+++ b/pym/portage/util/locale.py
@@ -6,7 +6,7 @@ Function to check whether the current used LC_CTYPE handles case
transformations of ASCII characters in a way compatible with the POSIX
locale.
"""
-from __future__ import unicode_literals
+from __future__ import absolute_import, unicode_literals
import locale
import logging
@@ -26,7 +26,6 @@ locale_categories = (
'LC_PAPER', 'LC_TELEPHONE',
)
-
_check_locale_cache = {}
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 90ccd027ee56f4ff0afaf67825aceea92ce633a5
Author: Michał Górny gentoo org>
AuthorDate: Sun Nov 15 22:49:33 2015 +
Commit: Michał Górny gentoo org>
CommitDate: Sun Nov 15 22:49:33 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=90ccd027
util/locale: Cache check_locale() results for specific locales
pym/portage/util/locale.py | 30 +-
1 file changed, 21 insertions(+), 9 deletions(-)
diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
index 27a2806..58ecb2a 100644
--- a/pym/portage/util/locale.py
+++ b/pym/portage/util/locale.py
@@ -27,6 +27,9 @@ locale_categories = (
)
+_check_locale_cache = {}
+
+
def _check_locale(silent):
"""
The inner locale check function.
@@ -82,17 +85,22 @@ def check_locale(silent=False, env=None):
can not be executed due to platform limitations.
"""
+ if env is not None:
+ for v in ("LC_ALL", "LC_CTYPE", "LANG"):
+ if v in env:
+ mylocale = env[v]
+ break
+ else:
+ mylocale = "C"
+
+ try:
+ return _check_locale_cache[mylocale]
+ except KeyError:
+
pid = os.fork()
if pid == 0:
try:
if env is not None:
- for v in ("LC_ALL", "LC_CTYPE", "LANG"):
- if v in env:
- mylocale = env[v]
- break
- else:
- mylocale = "C"
-
try:
locale.setlocale(locale.LC_CTYPE,
mylocale)
except locale.Error:
@@ -109,11 +117,15 @@ def check_locale(silent=False, env=None):
pid2, ret = os.waitpid(pid, 0)
assert pid == pid2
+ pyret = None
if os.WIFEXITED(ret):
ret = os.WEXITSTATUS(ret)
if ret != 2:
- return ret == 0
- return None
+ pyret = ret == 0
+
+ if env is not None:
+ _check_locale_cache[mylocale] = pyret
+ return pyret
def split_LC_ALL(env):
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: b7a07159e472912170d69d3520b188115b810561
Author: Zac Medico gentoo org>
AuthorDate: Mon Nov 16 01:28:55 2015 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Nov 16 01:29:23 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7a07159
util/locale: Fix IndentationError in check_locale
Fixes: 90ccd027ee56 ("util/locale: Cache check_locale() results for specific
locales")
pym/portage/util/locale.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
index 58ecb2a..c3332ea 100644
--- a/pym/portage/util/locale.py
+++ b/pym/portage/util/locale.py
@@ -96,6 +96,7 @@ def check_locale(silent=False, env=None):
try:
return _check_locale_cache[mylocale]
except KeyError:
+ pass
pid = os.fork()
if pid == 0:
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/, bin/
commit: a82dfe797defc1908bd9f97c1118b478994f6444
Author: Zac Medico gentoo org>
AuthorDate: Thu Nov 12 00:08:20 2015 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Nov 12 18:54:55 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a82dfe79
egencache: parallelize --update-changelogs (bug 565540)
Use the TaskScheduler class to parallelize GenChangeLogs. Fix
AsyncFunction so it does not re-define 'args' in __slots__.
X-Gentoo-Bug: 565540
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=565540
Acked-by: Brian Dolbec gentoo.org>
bin/egencache| 24 +++-
pym/portage/util/_async/AsyncFunction.py | 5 -
2 files changed, 23 insertions(+), 6 deletions(-)
diff --git a/bin/egencache b/bin/egencache
index 51d115a..76eb00b 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -55,7 +55,9 @@ from portage.const import TIMESTAMP_FORMAT
from portage.manifest import guessManifestFileType
from portage.package.ebuild._parallel_manifest.ManifestScheduler import
ManifestScheduler
from portage.util import cmp_sort_key, writemsg_level
+from portage.util._async.AsyncFunction import AsyncFunction
from portage.util._async.run_main_scheduler import run_main_scheduler
+from portage.util._async.TaskScheduler import TaskScheduler
from portage.util._eventloop.global_event_loop import global_event_loop
from portage import cpv_getkey
from portage.dep import Atom, isjustname
@@ -748,7 +750,8 @@ class _special_filename(_filename_base):
return self.file_name < other.file_name
class GenChangeLogs(object):
- def __init__(self, portdb, changelog_output, changelog_reversed):
+ def __init__(self, portdb, changelog_output, changelog_reversed,
+ max_jobs=None, max_load=None):
self.returncode = os.EX_OK
self._portdb = portdb
self._wrapper = textwrap.TextWrapper(
@@ -758,6 +761,8 @@ class GenChangeLogs(object):
)
self._changelog_output = changelog_output
self._changelog_reversed = changelog_reversed
+ self._max_jobs = max_jobs
+ self._max_load = max_load
@staticmethod
def grab(cmd):
@@ -882,7 +887,7 @@ class GenChangeLogs(object):
output.close()
- def run(self):
+ def _task_iter(self):
repo_path = self._portdb.porttrees[0]
os.chdir(repo_path)
@@ -908,7 +913,12 @@ class GenChangeLogs(object):
cmod = 0
if float(cmod) < float(lmod):
- self.generate_changelog(cp)
+ yield
AsyncFunction(target=self.generate_changelog, args=[cp])
+
+ def run(self):
+ return run_main_scheduler(
+ TaskScheduler(self._task_iter(),
event_loop=global_event_loop(),
+ max_jobs=self._max_jobs,
max_load=self._max_load))
def egencache_main(args):
@@ -1149,8 +1159,12 @@ def egencache_main(args):
if options.update_changelogs:
gen_clogs = GenChangeLogs(portdb,
changelog_output=options.changelog_output,
- changelog_reversed=options.changelog_reversed)
- gen_clogs.run()
+ changelog_reversed=options.changelog_reversed,
+ max_jobs=options.jobs,
+ max_load=options.load_average)
+ signum = gen_clogs.run()
+ if signum is not None:
+ sys.exit(128 + signum)
ret.append(gen_clogs.returncode)
if options.write_timestamp:
diff --git a/pym/portage/util/_async/AsyncFunction.py
b/pym/portage/util/_async/AsyncFunction.py
index b6142a2..40f6c5e 100644
--- a/pym/portage/util/_async/AsyncFunction.py
+++ b/pym/portage/util/_async/AsyncFunction.py
@@ -15,7 +15,10 @@ class AsyncFunction(ForkProcess):
"result" attribute after the forked process has exited.
"""
- __slots__ = ('args', 'kwargs', 'result', 'target',
+ # NOTE: This class overrides the meaning of the SpawnProcess 'args'
+ # attribute, and uses it to hold the positional arguments for the
+ # 'target' function.
+ __slots__ = ('kwargs', 'result', 'target',
'_async_func_reader', '_async_func_reader_pw')
def _start(self):
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 597987aac1677e132b80ed2404697acf0188af7a
Author: Zac Medico gentoo org>
AuthorDate: Tue Oct 6 20:14:54 2015 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Oct 6 20:23:07 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=597987aa
apply_secpass_permissions: avoid accessing portage.data.secpass when possible
This fixes PermissionDenied errors triggered when portage config files
and repos have restricted access permissions.
Fixes: b7baeeec3ab6 ("unpack: use chmod-lite helper for bug 554084")
pym/portage/util/__init__.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py
index 2b7ff8d..b739257 100644
--- a/pym/portage/util/__init__.py
+++ b/pym/portage/util/__init__.py
@@ -1241,7 +1241,9 @@ def apply_secpass_permissions(filename, uid=-1, gid=-1,
mode=-1, mask=-1,
all_applied = True
- if portage.data.secpass < 2:
+ # Avoid accessing portage.data.secpass when possible, since
+ # it triggers config loading (undesirable for chmod-lite).
+ if (uid != -1 or gid != -1) and portage.data.secpass < 2:
if uid != -1 and \
uid != stat_cached.st_uid:
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: c363731ecf59f74386001ba56ba0d7a73de7f2f4 Author: Zac Medico zmedico AT gentoo DOT org AuthorDate: Sun May 31 20:51:42 2015 + Commit: Zac Medico zmedico AT gentoo DOT org CommitDate: Fri Jun 5 01:30:06 2015 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c363731e movefile: enable absolute_import for Python 2 (bug 550886) Since commit 1032cbf4c218741df1c57767fead2d00cc6321d9, with Python 2, movefile imports portage.util.xattr instead of xattr. Fix it by enabling absolute_import. Fixes: 1032cbf4c218 (quickpkg: support FEATURES=xattr (bug 550006)) X-Gentoo-Bug: 550886 X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=550886 pym/portage/util/movefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/portage/util/movefile.py b/pym/portage/util/movefile.py index d00f624..1000569 100644 --- a/pym/portage/util/movefile.py +++ b/pym/portage/util/movefile.py @@ -1,7 +1,7 @@ # Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals +from __future__ import absolute_import, unicode_literals __all__ = ['movefile']
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
commit: c3ef903d1c95ce28922a2d7dcb00cc7a1d690201
Author: Zac Medico zmedico AT gentoo DOT org
AuthorDate: Sun May 31 18:42:49 2015 +
Commit: Zac Medico zmedico AT gentoo DOT org
CommitDate: Tue Jun 2 03:47:14 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c3ef903d
_approx_multilib_categories: rename ia to ia64 (bug 550898)
In LinkageMapELF.py, there's a fallback mapping for
multilib categories, which is used by the LinkMapElf class when
NEEDED.ELF.2 does not contain multilib categories due to being
generated by older portage. This mapping should be consistent with
the mutilib categories generated by the compute_multilib_category
function which was modified by commit
b50bbcb70506254d66937dac376056bbf99b3fe9.
Fixes: b50bbcb70506 (multilib: use ia64 as ia64 multilib name)
X-Gentoo-Bug: 550898
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=550898
Acked-by: Mike Frysinger vapier AT gentoo.org
pym/portage/util/_dyn_libs/LinkageMapELF.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py
b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index f4d8b5d..63e2213 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -30,7 +30,7 @@ _approx_multilib_categories = {
AARCH64: arm_64,
ALPHA: alpha_64,
ARM: arm_32,
- IA_64: ia_64,
+ IA_64: ia64_64,
MIPS: mips_o32,
PARISC:hppa_64,
PPC: ppc_32,
[gentoo-commits] proj/portage:master commit in: pym/portage/util/, bin/, pym/_emerge/, pym/portage/
commit: 971802f170e43fd36241decfa63e527dc1b3cf4f Author: Zac Medico zmedico AT gentoo DOT org AuthorDate: Sat Apr 25 20:31:15 2015 + Commit: Zac Medico zmedico AT gentoo DOT org CommitDate: Wed May 6 18:19:01 2015 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=971802f1 Bundle a minimalistic derivation of Python's formatter module (bug 547732) Python's formatter module is scheduled for removal in Python 3.6, so replace it with a minimalistic derivation. X-Gentoo-Bug: 547732 X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=547732 Acked-by: Brian Dolbec dolsen AT gentoo.org bin/repoman | 2 +- pym/_emerge/JobStatusDisplay.py | 4 +-- pym/portage/output.py | 4 +-- pym/portage/util/formatter.py | 69 + 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/bin/repoman b/bin/repoman index 37e11b2..a758aea 100755 --- a/bin/repoman +++ b/bin/repoman @@ -11,7 +11,6 @@ from __future__ import print_function, unicode_literals import codecs import copy import errno -import formatter import io import logging import re @@ -56,6 +55,7 @@ except (ImportError, SystemError, RuntimeError, Exception): from portage import os from portage import _encodings from portage import _unicode_encode +import portage.util.formatter as formatter import repoman.checks from repoman.checks import run_checks from repoman.check_missingslot import check_missingslot diff --git a/pym/_emerge/JobStatusDisplay.py b/pym/_emerge/JobStatusDisplay.py index 9f6f09b..b8e142a 100644 --- a/pym/_emerge/JobStatusDisplay.py +++ b/pym/_emerge/JobStatusDisplay.py @@ -1,14 +1,14 @@ -# Copyright 1999-2013 Gentoo Foundation +# Copyright 1999-2015 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals -import formatter import io import sys import time import portage +import portage.util.formatter as formatter from portage import os from portage import _encodings from portage import _unicode_encode diff --git a/pym/portage/output.py b/pym/portage/output.py index 7846627..bb7542b 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -1,4 +1,4 @@ -# Copyright 1998-2014 Gentoo Foundation +# Copyright 1998-2015 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import division @@ -7,7 +7,6 @@ __docformat__ = epytext import errno import io -import formatter import re import subprocess import sys @@ -16,6 +15,7 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:writemsg', ) +import portage.util.formatter as formatter from portage import os from portage import _encodings diff --git a/pym/portage/util/formatter.py b/pym/portage/util/formatter.py new file mode 100644 index 000..ce6799e --- /dev/null +++ b/pym/portage/util/formatter.py @@ -0,0 +1,69 @@ +# Copyright 2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# +# This is a minimalistic derivation of Python's deprecated formatter module, +# supporting only the methods related to style, literal data, and line breaks. + +import sys + + +class AbstractFormatter(object): + The standard formatter. + + def __init__(self, writer): + self.writer = writer# Output device + self.style_stack = [] # Other state, e.g. color + self.hard_break = True # Have a hard break + + def add_line_break(self): + if not self.hard_break: + self.writer.send_line_break() + self.hard_break = True + + def add_literal_data(self, data): + if not data: return + self.hard_break = data[-1:] == '\n' + self.writer.send_literal_data(data) + + def push_style(self, *styles): + for style in styles: + self.style_stack.append(style) + self.writer.new_styles(tuple(self.style_stack)) + + def pop_style(self, n=1): + del self.style_stack[-n:] + self.writer.new_styles(tuple(self.style_stack)) + + +class NullWriter(object): + Minimal writer interface to use in testing inheritance. + + A writer which only provides the interface definition; no actions are + taken on any methods. This should be the base class for all writers + which do not need to inherit any implementation methods. + + def __init__(self): pass + def flush(self): pass + def new_styles(self, styles): pass + def send_line_break(self): pass + def send_literal_data(self, data): pass + + +class DumbWriter(NullWriter): + Simple writer class which writes output on the file object passed in + as the file parameter or, if file is omitted, on standard output. + + +
[gentoo-commits] proj/portage:master commit in: pym/portage/util/
commit: 1375a55bd6f9a35ee1a4b4ec78b84f830cfb41a9
Author: Zac Medico zmedico AT gentoo DOT org
AuthorDate: Mon May 4 05:55:20 2015 +
Commit: Zac Medico zmedico AT gentoo DOT org
CommitDate: Mon May 4 06:41:58 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1375a55b
varexpand: fix IndexError (bug 548556)
This handles two cases where varexpand incremented the index without
checking bounds.
X-Gentoo-Bug: 548556
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=548556
X-Gentoo-forum-thread: https://forums.gentoo.org/viewtopic-t-1016432.html
Acked-by: Brian Dolbec dolsen AT gentoo.org
pym/portage/util/__init__.py | 12
1 file changed, 12 insertions(+)
diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py
index 48cd1b7..c0b509b 100644
--- a/pym/portage/util/__init__.py
+++ b/pym/portage/util/__init__.py
@@ -850,8 +850,20 @@ def varexpand(mystring, mydict=None, error_leader=None):
continue
elif current == $:
pos += 1
+ if pos == length:
+ # shells handle this like \$
+ newstring.append(current)
+ continue
+
if mystring[pos] == {:
pos += 1
+ if pos == length:
+ msg =
_varexpand_unexpected_eof_msg
+ if error_leader is not None:
+ msg = error_leader() +
msg
+ writemsg(msg + \n,
noiselevel=-1)
+ return
+
braced = True
else:
braced = False
[gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
commit: a47407e864d9a5a43d6f05ef06b46e34ff0e0c24
Author: Zac Medico zmedico AT gentoo DOT org
AuthorDate: Thu Apr 23 17:47:03 2015 +
Commit: Zac Medico zmedico AT gentoo DOT org
CommitDate: Fri Apr 24 17:51:45 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a47407e8
LinkageMapElf.rebuild: pass error_leader to varexpand (bug 542796)
Since commit f1c1b8a77eebf7713b32e5f9945690f60f4f46de,
LinkageMapElf.rebuild could produce mysterious bad substitution
messages.
Fixes: f1c1b8a77eeb (Generate soname dependency metadata (bug 282639))
X-Gentoo-Bug: 542796
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=542796
X-Gentoo-forum-thread: https://forums.gentoo.org/viewtopic-t-1014842.html
Acked-by: Brian Dolbec dolsen AT gentoo.org
pym/portage/util/_dyn_libs/LinkageMapELF.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py
b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index c44666a..f4d8b5d 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -339,7 +339,8 @@ class LinkageMapELF(object):
obj = entry.filename
soname = entry.soname
expand = {ORIGIN: os.path.dirname(entry.filename)}
- path = frozenset(normalize_path(varexpand(x, expand))
+ path = frozenset(normalize_path(
+ varexpand(x, expand, error_leader=lambda: %s:
% location))
for x in entry.runpaths)
path = frozensets.setdefault(path, path)
needed = frozenset(entry.needed)
