[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/, ...
commit: b2d8226af4589d95f44d20d441056f645a523039
Author: Zac Medico gentoo org>
AuthorDate: Thu Feb 29 04:26:02 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Feb 29 04:37:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2d8226a
Delete compat_coroutine module
The compat_coroutine module has been unused since the migration
to PEP 492 async and await syntax in 2021, which began in commit
b3b9acc13c43 and was completed in commit bcda30d0a6fa.
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/futures/meson.build | 1 -
.../tests/util/futures/test_compat_coroutine.py| 210 -
lib/portage/util/futures/_asyncio/__init__.py | 15 +-
lib/portage/util/futures/compat_coroutine.py | 142 --
lib/portage/util/futures/meson.build | 1 -
5 files changed, 1 insertion(+), 368 deletions(-)
diff --git a/lib/portage/tests/util/futures/meson.build
b/lib/portage/tests/util/futures/meson.build
index 877acc27cd..cb78314844 100644
--- a/lib/portage/tests/util/futures/meson.build
+++ b/lib/portage/tests/util/futures/meson.build
@@ -1,6 +1,5 @@
py.install_sources(
[
-'test_compat_coroutine.py',
'test_done_callback.py',
'test_done_callback_after_exit.py',
'test_iter_completed.py',
diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py
b/lib/portage/tests/util/futures/test_compat_coroutine.py
deleted file mode 100644
index b25708886c..00
--- a/lib/portage/tests/util/futures/test_compat_coroutine.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Copyright 2018 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import (
-coroutine,
-coroutine_return,
-)
-from portage.util.futures._sync_decorator import _sync_decorator, _sync_methods
-from portage.tests import TestCase
-
-
-class CompatCoroutineTestCase(TestCase):
-def test_returning_coroutine(self):
-@coroutine
-def returning_coroutine(loop=None):
-yield asyncio.sleep(0, loop=loop)
-coroutine_return("success")
-
-loop = asyncio.get_event_loop()
-self.assertEqual(
-"success",
-
asyncio.get_event_loop().run_until_complete(returning_coroutine(loop=loop)),
-)
-
-def test_raising_coroutine(self):
-class TestException(Exception):
-pass
-
-@coroutine
-def raising_coroutine(loop=None):
-yield asyncio.sleep(0, loop=loop)
-raise TestException("exception")
-
-loop = asyncio.get_event_loop()
-self.assertRaises(
-TestException, loop.run_until_complete,
raising_coroutine(loop=loop)
-)
-
-def test_catching_coroutine(self):
-class TestException(Exception):
-pass
-
-@coroutine
-def catching_coroutine(loop=None):
-loop = asyncio._wrap_loop(loop)
-future = loop.create_future()
-loop.call_soon(future.set_exception, TestException("exception"))
-try:
-yield future
-except TestException:
-self.assertTrue(True)
-else:
-self.assertTrue(False)
-coroutine_return("success")
-
-loop = asyncio.get_event_loop()
-self.assertEqual(
-"success", loop.run_until_complete(catching_coroutine(loop=loop))
-)
-
-def test_cancelled_coroutine(self):
-"""
-Verify that a coroutine can handle (and reraise) asyncio.CancelledError
-in order to perform any necessary cleanup. Note that the
-asyncio.CancelledError will only be thrown in the coroutine if there's
-an opportunity (yield) before the generator raises StopIteration.
-"""
-loop = asyncio.get_event_loop()
-ready_for_exception = loop.create_future()
-exception_in_coroutine = loop.create_future()
-
-@coroutine
-def cancelled_coroutine(loop=None):
-loop = asyncio._wrap_loop(loop)
-while True:
-task = loop.create_future()
-try:
-ready_for_exception.set_result(None)
-yield task
-except BaseException as e:
-# Since python3.8, asyncio.CancelledError inherits
-# from BaseException.
-task.done() or task.cancel()
-exception_in_coroutine.set_exception(e)
-raise
-else:
-exception_in_coroutine.set_result(None)
-
-future = cancelled_coroutine(loop=loop)
-loop.run_until_complete(ready_for_exception)
-future.cancel()
-
-self.assertRaises(asyncio.CancelledError, loop.run_until_complete,
future)
-
-
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/, ...
commit: bf1505777847aface5b4cac16b955071c2915ddd
Author: Zac Medico gentoo org>
AuthorDate: Wed Apr 8 05:03:34 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Apr 8 05:29:48 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bf150577
Revert "AsyncScheduler: use async_start method"
This reverts commit 8f47d3fe1190d4476ae9eebfafcebdfb1794fc05.
Bug: https://bugs.gentoo.org/716636
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 8 +++--
.../tests/util/futures/test_iter_completed.py | 2 --
lib/portage/util/_async/AsyncScheduler.py | 20 ++--
lib/portage/util/futures/iter_completed.py | 38 +-
4 files changed, 15 insertions(+), 53 deletions(-)
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index 50fc5fe1c..05ea24c4b 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -109,16 +109,18 @@ class DoebuildFdPipesTestCase(TestCase):
output_fd: pw,
},
"prev_mtimes": {}})
- producer.addStartListener(lambda producer:
os.close(pw))
- # PipeReader closes pr
consumer = PipeReader(
input_files={"producer" : pr})
task_scheduler = TaskScheduler(iter([producer,
consumer]),
max_jobs=2)
-
loop.run_until_complete(task_scheduler.async_start())
+ try:
+
loop.run_until_complete(task_scheduler.async_start())
+ finally:
+ # PipeReader closes pr
+ os.close(pw)
task_scheduler.wait()
output = portage._unicode_decode(
diff --git a/lib/portage/tests/util/futures/test_iter_completed.py
b/lib/portage/tests/util/futures/test_iter_completed.py
index 03ace915a..aa24f5685 100644
--- a/lib/portage/tests/util/futures/test_iter_completed.py
+++ b/lib/portage/tests/util/futures/test_iter_completed.py
@@ -76,8 +76,6 @@ class IterCompletedTestCase(TestCase):
for future_done_set in async_iter_completed(future_generator(),
max_jobs=True, max_load=True, loop=loop):
- while not input_futures:
- loop.run_until_complete(asyncio.sleep(0,
loop=loop))
future_done_set.cancel()
break
diff --git a/lib/portage/util/_async/AsyncScheduler.py
b/lib/portage/util/_async/AsyncScheduler.py
index b9070061a..c6b523eaa 100644
--- a/lib/portage/util/_async/AsyncScheduler.py
+++ b/lib/portage/util/_async/AsyncScheduler.py
@@ -1,11 +1,7 @@
# Copyright 2012-2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-import functools
-
from portage import os
-from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import coroutine
from _emerge.AsynchronousTask import AsynchronousTask
from _emerge.PollScheduler import PollScheduler
@@ -66,8 +62,8 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
else:
self._running_tasks.add(task)
task.scheduler = self._sched_iface
- future =
asyncio.ensure_future(self._task_coroutine(task), loop=self._sched_iface)
-
future.add_done_callback(functools.partial(self._task_coroutine_done, task))
+ task.addExitListener(self._task_exit)
+ task.start()
if self._loadavg_check_id is not None:
self._loadavg_check_id.cancel()
@@ -77,18 +73,6 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
# Triggers cleanup and exit listeners if there's nothing left
to do.
self.poll()
- @coroutine
- def _task_coroutine(self, task):
- yield task.async_start()
- yield task.async_wait()
-
- def _task_coroutine_done(self, task, future):
- try:
- future.result()
- except asyncio.CancelledError:
- self.cancel()
- self._task_exit(task)
-
def _task_exit(self, task):
self._running_tasks.discard(task)
if task.returncode != os.EX_OK:
diff --git
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/, lib/_emerge/
commit: bab11fcee344df488d2e7f444ea3711ce87669e3
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 1 21:56:41 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Mar 2 00:35:51 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bab11fce
_GeneratorTask: throw CancelledError in cancelled coroutine (bug 711174)
Throw asyncio.CancelledError in a cancelled coroutine, ensuring
that the coroutine can handle this exception in order to perform
any necessary cleanup (like close the log file for bug 711174).
Note that the asyncio.CancelledError will only be thrown in the
coroutine if there's an opportunity (yield) before the generator
raises StopIteration.
Also fix the AsynchronousTask exit listener handling for
compatibility with this new behavior.
Fixes: 8074127bbc21 ("SpawnProcess: add _main coroutine")
Bug: https://bugs.gentoo.org/711174
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/AsynchronousTask.py| 12 ++---
.../tests/util/futures/test_compat_coroutine.py| 29 +++---
lib/portage/util/futures/compat_coroutine.py | 19 ++
3 files changed, 49 insertions(+), 11 deletions(-)
diff --git a/lib/_emerge/AsynchronousTask.py b/lib/_emerge/AsynchronousTask.py
index 1e9e177cb..580eef050 100644
--- a/lib/_emerge/AsynchronousTask.py
+++ b/lib/_emerge/AsynchronousTask.py
@@ -64,7 +64,7 @@ class AsynchronousTask(SlotObject):
@returns: Future, result is self.returncode
"""
waiter = self.scheduler.create_future()
- exit_listener = lambda self: waiter.set_result(self.returncode)
+ exit_listener = lambda self: waiter.cancelled() or
waiter.set_result(self.returncode)
self.addExitListener(exit_listener)
waiter.add_done_callback(lambda waiter:
self.removeExitListener(exit_listener) if
waiter.cancelled() else None)
@@ -180,9 +180,15 @@ class AsynchronousTask(SlotObject):
def removeExitListener(self, f):
if self._exit_listeners is None:
if self._exit_listener_stack is not None:
- self._exit_listener_stack.remove(f)
+ try:
+ self._exit_listener_stack.remove(f)
+ except ValueError:
+ pass
return
- self._exit_listeners.remove(f)
+ try:
+ self._exit_listeners.remove(f)
+ except ValueError:
+ pass
def _wait_hook(self):
"""
diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py
b/lib/portage/tests/util/futures/test_compat_coroutine.py
index f96aa9be5..b561c0227 100644
--- a/lib/portage/tests/util/futures/test_compat_coroutine.py
+++ b/lib/portage/tests/util/futures/test_compat_coroutine.py
@@ -57,20 +57,43 @@ class CompatCoroutineTestCase(TestCase):
loop.run_until_complete(catching_coroutine(loop=loop)))
def test_cancelled_coroutine(self):
+ """
+ Verify that a coroutine can handle (and reraise)
asyncio.CancelledError
+ in order to perform any necessary cleanup. Note that the
+ asyncio.CancelledError will only be thrown in the coroutine if
there's
+ an opportunity (yield) before the generator raises
StopIteration.
+ """
+ loop = asyncio.get_event_loop()
+ ready_for_exception = loop.create_future()
+ exception_in_coroutine = loop.create_future()
@coroutine
def cancelled_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
while True:
- yield loop.create_future()
+ task = loop.create_future()
+ try:
+ ready_for_exception.set_result(None)
+ yield task
+ except BaseException as e:
+ # Since python3.8,
asyncio.CancelledError inherits
+ # from BaseException.
+ task.done() or task.cancel()
+ exception_in_coroutine.set_exception(e)
+ raise
+ else:
+ exception_in_coroutine.set_result(None)
- loop = asyncio.get_event_loop()
future = cancelled_coroutine(loop=loop)
- loop.call_soon(future.cancel)
+ loop.run_until_complete(ready_for_exception)
+ future.cancel()
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/
commit: de0b60ff277311e780102131dce3111b4db1c196
Author: Zac Medico gentoo org>
AuthorDate: Sat Jul 28 13:53:11 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Sep 24 03:41:32 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=de0b60ff
Add _sync_decorator module
Add functions that decorate coroutine methods and functions for
synchronous usage, allowing coroutines to smoothly blend with
synchronous code. This eliminates clutter that might otherwise
discourage the proliferation of coroutine usage for I/O bound tasks.
In the next commit, _sync_decorator will be used for smooth
integration of new classes that have coroutine methods.
Bug: https://bugs.gentoo.org/662070
Reviewed-by: Brian Dolbec gentoo.org>
Signed-off-by: Zac Medico gentoo.org>
.../tests/util/futures/test_compat_coroutine.py| 14 ++
lib/portage/util/futures/_sync_decorator.py| 54 ++
2 files changed, 68 insertions(+)
diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py
b/lib/portage/tests/util/futures/test_compat_coroutine.py
index b6f75b1a2..f96aa9be5 100644
--- a/lib/portage/tests/util/futures/test_compat_coroutine.py
+++ b/lib/portage/tests/util/futures/test_compat_coroutine.py
@@ -6,6 +6,7 @@ from portage.util.futures.compat_coroutine import (
coroutine,
coroutine_return,
)
+from portage.util.futures._sync_decorator import _sync_decorator, _sync_methods
from portage.tests import TestCase
@@ -161,3 +162,16 @@ class CompatCoroutineTestCase(TestCase):
loop.run_until_complete(asyncio.wait([writer, reader]))
self.assertEqual(reader.result(), values)
+
+ # Test decoration of coroutine methods and functions for
+ # synchronous usage, allowing coroutines to smoothly
+ # blend with synchronous code.
+ sync_cubby = _sync_methods(cubby, loop=loop)
+ sync_reader = _sync_decorator(reader_coroutine, loop=loop)
+ writer = asyncio.ensure_future(writer_coroutine(cubby, values,
None), loop=loop)
+ self.assertEqual(sync_reader(cubby, None), values)
+ self.assertTrue(writer.done())
+
+ for i in range(3):
+ sync_cubby.write(i)
+ self.assertEqual(sync_cubby.read(), i)
diff --git a/lib/portage/util/futures/_sync_decorator.py
b/lib/portage/util/futures/_sync_decorator.py
new file mode 100644
index 0..02a0963a7
--- /dev/null
+++ b/lib/portage/util/futures/_sync_decorator.py
@@ -0,0 +1,54 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import functools
+
+import portage
+portage.proxy.lazyimport.lazyimport(globals(),
+ 'portage.util.futures:asyncio',
+)
+
+
+def _sync_decorator(func, loop=None):
+ """
+ Decorate an asynchronous function (either a corouting function or a
+ function that returns a Future) with a wrapper that runs the function
+ synchronously.
+ """
+ loop = asyncio._wrap_loop(loop)
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ return loop.run_until_complete(func(*args, **kwargs))
+ return wrapper
+
+
+def _sync_methods(obj, loop=None):
+ """
+ For use with synchronous code that needs to interact with an object
+ that has coroutine methods, this function generates a proxy which
+ conveniently converts coroutine methods into synchronous methods.
+ This allows coroutines to smoothly blend with synchronous
+ code, eliminating clutter that might otherwise discourage the
+ proliferation of coroutine usage for I/O bound tasks.
+ """
+ loop = asyncio._wrap_loop(loop)
+ return _ObjectAttrWrapper(obj,
+ lambda attr: _sync_decorator(attr, loop=loop)
+ if asyncio.iscoroutinefunction(attr) else attr)
+
+
+class _ObjectAttrWrapper(portage.proxy.objectproxy.ObjectProxy):
+
+ __slots__ = ('_obj', '_attr_wrapper')
+
+ def __init__(self, obj, attr_wrapper):
+ object.__setattr__(self, '_obj', obj)
+ object.__setattr__(self, '_attr_wrapper', attr_wrapper)
+
+ def __getattribute__(self, attr):
+ obj = object.__getattribute__(self, '_obj')
+ attr_wrapper = object.__getattribute__(self, '_attr_wrapper')
+ return attr_wrapper(getattr(obj, attr))
+
+ def _get_target(self):
+ return object.__getattribute__(self, '_obj')
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/
commit: b37256a524a0fbf88ffad20c9f01aaf37409ec66 Author: Zac Medico gentoo org> AuthorDate: Mon Sep 3 20:16:37 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Sep 3 20:20:40 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b37256a5 compat_coroutine: CancelledError cancels coroutine's future lib/portage/tests/util/futures/test_compat_coroutine.py | 8 ++-- lib/portage/util/futures/compat_coroutine.py| 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py b/lib/portage/tests/util/futures/test_compat_coroutine.py index cbc070869..b6f75b1a2 100644 --- a/lib/portage/tests/util/futures/test_compat_coroutine.py +++ b/lib/portage/tests/util/futures/test_compat_coroutine.py @@ -71,6 +71,10 @@ class CompatCoroutineTestCase(TestCase): loop.run_until_complete, future) def test_cancelled_future(self): + """ + When a coroutine raises CancelledError, the coroutine's + future is cancelled. + """ @coroutine def cancelled_future_coroutine(loop=None): @@ -81,8 +85,8 @@ class CompatCoroutineTestCase(TestCase): yield future loop = asyncio.get_event_loop() - self.assertRaises(asyncio.CancelledError, - loop.run_until_complete, cancelled_future_coroutine(loop=loop)) + future = loop.run_until_complete(asyncio.wait([cancelled_future_coroutine()]))[0].pop() + self.assertTrue(future.cancelled()) def test_yield_expression_result(self): @coroutine diff --git a/lib/portage/util/futures/compat_coroutine.py b/lib/portage/util/futures/compat_coroutine.py index 59fdc31b6..3edfa6bee 100644 --- a/lib/portage/util/futures/compat_coroutine.py +++ b/lib/portage/util/futures/compat_coroutine.py @@ -102,6 +102,8 @@ class _GeneratorTask(object): self._generator.throw(previous.exception()) future = next(self._generator) + except asyncio.CancelledError: + self._result.cancel() except _CoroutineReturnValue as e: if not self._result.cancelled(): self._result.set_result(e.result)
