[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/, ...
commit: b2d8226af4589d95f44d20d441056f645a523039
Author: Zac Medico gentoo org>
AuthorDate: Thu Feb 29 04:26:02 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Feb 29 04:37:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2d8226a
Delete compat_coroutine module
The compat_coroutine module has been unused since the migration
to PEP 492 async and await syntax in 2021, which began in commit
b3b9acc13c43 and was completed in commit bcda30d0a6fa.
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/futures/meson.build | 1 -
.../tests/util/futures/test_compat_coroutine.py| 210 -
lib/portage/util/futures/_asyncio/__init__.py | 15 +-
lib/portage/util/futures/compat_coroutine.py | 142 --
lib/portage/util/futures/meson.build | 1 -
5 files changed, 1 insertion(+), 368 deletions(-)
diff --git a/lib/portage/tests/util/futures/meson.build
b/lib/portage/tests/util/futures/meson.build
index 877acc27cd..cb78314844 100644
--- a/lib/portage/tests/util/futures/meson.build
+++ b/lib/portage/tests/util/futures/meson.build
@@ -1,6 +1,5 @@
py.install_sources(
[
-'test_compat_coroutine.py',
'test_done_callback.py',
'test_done_callback_after_exit.py',
'test_iter_completed.py',
diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py
b/lib/portage/tests/util/futures/test_compat_coroutine.py
deleted file mode 100644
index b25708886c..00
--- a/lib/portage/tests/util/futures/test_compat_coroutine.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Copyright 2018 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import (
-coroutine,
-coroutine_return,
-)
-from portage.util.futures._sync_decorator import _sync_decorator, _sync_methods
-from portage.tests import TestCase
-
-
-class CompatCoroutineTestCase(TestCase):
-def test_returning_coroutine(self):
-@coroutine
-def returning_coroutine(loop=None):
-yield asyncio.sleep(0, loop=loop)
-coroutine_return("success")
-
-loop = asyncio.get_event_loop()
-self.assertEqual(
-"success",
-
asyncio.get_event_loop().run_until_complete(returning_coroutine(loop=loop)),
-)
-
-def test_raising_coroutine(self):
-class TestException(Exception):
-pass
-
-@coroutine
-def raising_coroutine(loop=None):
-yield asyncio.sleep(0, loop=loop)
-raise TestException("exception")
-
-loop = asyncio.get_event_loop()
-self.assertRaises(
-TestException, loop.run_until_complete,
raising_coroutine(loop=loop)
-)
-
-def test_catching_coroutine(self):
-class TestException(Exception):
-pass
-
-@coroutine
-def catching_coroutine(loop=None):
-loop = asyncio._wrap_loop(loop)
-future = loop.create_future()
-loop.call_soon(future.set_exception, TestException("exception"))
-try:
-yield future
-except TestException:
-self.assertTrue(True)
-else:
-self.assertTrue(False)
-coroutine_return("success")
-
-loop = asyncio.get_event_loop()
-self.assertEqual(
-"success", loop.run_until_complete(catching_coroutine(loop=loop))
-)
-
-def test_cancelled_coroutine(self):
-"""
-Verify that a coroutine can handle (and reraise) asyncio.CancelledError
-in order to perform any necessary cleanup. Note that the
-asyncio.CancelledError will only be thrown in the coroutine if there's
-an opportunity (yield) before the generator raises StopIteration.
-"""
-loop = asyncio.get_event_loop()
-ready_for_exception = loop.create_future()
-exception_in_coroutine = loop.create_future()
-
-@coroutine
-def cancelled_coroutine(loop=None):
-loop = asyncio._wrap_loop(loop)
-while True:
-task = loop.create_future()
-try:
-ready_for_exception.set_result(None)
-yield task
-except BaseException as e:
-# Since python3.8, asyncio.CancelledError inherits
-# from BaseException.
-task.done() or task.cancel()
-exception_in_coroutine.set_exception(e)
-raise
-else:
-exception_in_coroutine.set_result(None)
-
-future = cancelled_coroutine(loop=loop)
-loop.run_until_complete(ready_for_exception)
-future.cancel()
-
-self.assertRaises(asyncio.CancelledError, loop.run_until_complete,
future)
-
-
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/tests/sets/files/, ...
commit: 3110ec376cbcb1f5b7fb82ba30ec958798bb32cf
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 00:46:52 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 13 05:04:40 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3110ec37
actions: Fix interaction between start-method and pytest-xdist
Use portage.test.TestCase setUp method to setup the multiprocessing
start method if needed. It needs to be done relatively late in order
to work with the pytest-xdist plugin due to execnet usage.
Bug: https://bugs.gentoo.org/914876
Signed-off-by: Zac Medico gentoo.org>
.github/workflows/ci.yml| 2 +-
lib/portage/tests/__init__.py | 12 +++-
lib/portage/tests/env/config/test_PortageModulesFile.py | 3 ++-
lib/portage/tests/news/test_NewsItem.py | 3 ++-
lib/portage/tests/sets/files/test_config_file_set.py| 3 ++-
lib/portage/tests/sets/files/test_static_file_set.py| 3 ++-
lib/portage/tests/sets/shell/test_shell.py | 4 ++--
lib/portage/tests/util/futures/test_retry.py| 1 +
8 files changed, 23 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 15da507238..5bffd97206 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -82,7 +82,6 @@ jobs:
mv "${bin_file}"{.new,}
fi
done < <(find bin -maxdepth 1 -type f)
- sed -i meson.build -e "s|'-m', 'pytest'|'-c', 'import
multiprocessing, sys, pytest; multiprocessing.set_start_method(\"spawn\",
force=True); sys.exit(pytest.console_main())'|"
- name: Test meson install --destdir /tmp/install-root
run: |
echo -e "[binaries]\npython = '$(command -v python)'" >
/tmp/native.ini
@@ -90,5 +89,6 @@ jobs:
meson install -C /tmp/build --destdir /tmp/install-root
- name: Run tests for ${{ matrix.python-version }}
run: |
+ [[ "${{ matrix.start-method }}" == "spawn" ]] && export
PORTAGE_MULTIPROCESSING_START_METHOD=spawn
export PYTEST_ADDOPTS="-vv -ra -l -o console_output_style=count -n
$(nproc) --dist=worksteal"
meson test -C /tmp/build --verbose
diff --git a/lib/portage/tests/__init__.py b/lib/portage/tests/__init__.py
index ef59852989..23dd366d89 100644
--- a/lib/portage/tests/__init__.py
+++ b/lib/portage/tests/__init__.py
@@ -1,8 +1,9 @@
# tests/__init__.py -- Portage Unit Test functionality
-# Copyright 2006-2023 Gentoo Authors
+# Copyright 2006-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import argparse
+import multiprocessing
import sys
import time
import unittest
@@ -79,6 +80,15 @@ class TestCase(unittest.TestCase):
self.bindir = cnf_bindir
self.sbindir = cnf_sbindir
+def setUp(self):
+"""
+Setup multiprocessing start method if needed. It needs to be
+done relatively late in order to work with the pytest-xdist
+plugin due to execnet usage.
+"""
+if os.environ.get("PORTAGE_MULTIPROCESSING_START_METHOD") == "spawn":
+multiprocessing.set_start_method("spawn", force=True)
+
def assertRaisesMsg(self, msg, excClass, callableObj, *args, **kwargs):
"""Fail unless an exception of class excClass is thrown
by callableObj when invoked with arguments args and keyword
diff --git a/lib/portage/tests/env/config/test_PortageModulesFile.py
b/lib/portage/tests/env/config/test_PortageModulesFile.py
index f9879df687..bca86e0e6c 100644
--- a/lib/portage/tests/env/config/test_PortageModulesFile.py
+++ b/lib/portage/tests/env/config/test_PortageModulesFile.py
@@ -1,4 +1,4 @@
-# Copyright 2006-2009 Gentoo Foundation
+# Copyright 2006-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage import os
@@ -13,6 +13,7 @@ class PortageModulesFileTestCase(TestCase):
modules = ["spanky", "zmedico", "antarus", "ricer", "5", "6"]
def setUp(self):
+super().setUp()
self.items = {}
for k, v in zip(self.keys + self.invalid_keys, self.modules):
self.items[k] = v
diff --git a/lib/portage/tests/news/test_NewsItem.py
b/lib/portage/tests/news/test_NewsItem.py
index a7903f07e4..7a8393c51f 100644
--- a/lib/portage/tests/news/test_NewsItem.py
+++ b/lib/portage/tests/news/test_NewsItem.py
@@ -1,5 +1,5 @@
# test_NewsItem.py -- Portage Unit Testing Functionality
-# Copyright 2007-2023 Gentoo Authors
+# Copyright 2007-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -114,6 +114,7 @@ class NewsItemTestCase(TestCase):
}
def setUp(self) -> None:
+super().setUp()
self.profile_base = "/var/db/repos/gentoo/profiles/default-linux"
self.profile = f"{self.profile_base}/x86/2007.0/"
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/asyncio/
commit: 66d8f8388e5b9da7e07510f78ec487913e2ceaf5
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 4 00:16:29 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 7 00:36:17 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=66d8f838
ChildWatcherTestCase: Remove obsolete test which uses spawn returnpid
This test was added for bug 649588 when there was still an
internal event loop implementation for python2. It is no
longer relevant and uses the deprecated spawn returnpid
parameter, so remove it.
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/futures/asyncio/meson.build | 1 -
.../util/futures/asyncio/test_child_watcher.py | 50 --
2 files changed, 51 deletions(-)
diff --git a/lib/portage/tests/util/futures/asyncio/meson.build
b/lib/portage/tests/util/futures/asyncio/meson.build
index ba727052fc..2de0668d6b 100644
--- a/lib/portage/tests/util/futures/asyncio/meson.build
+++ b/lib/portage/tests/util/futures/asyncio/meson.build
@@ -1,6 +1,5 @@
py.install_sources(
[
-'test_child_watcher.py',
'test_event_loop_in_fork.py',
'test_pipe_closed.py',
'test_policy_wrapper_recursion.py',
diff --git a/lib/portage/tests/util/futures/asyncio/test_child_watcher.py
b/lib/portage/tests/util/futures/asyncio/test_child_watcher.py
deleted file mode 100644
index cd100598b7..00
--- a/lib/portage/tests/util/futures/asyncio/test_child_watcher.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Copyright 2018-2021 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-import os
-
-from portage.process import find_binary, spawn
-from portage.tests import TestCase
-from portage.util._eventloop.global_event_loop import global_event_loop
-from portage.util.futures import asyncio
-from portage.util.futures.unix_events import DefaultEventLoopPolicy
-
-
-class ChildWatcherTestCase(TestCase):
-def testChildWatcher(self):
-true_binary = find_binary("true")
-self.assertNotEqual(true_binary, None)
-
-initial_policy = asyncio.get_event_loop_policy()
-if not isinstance(initial_policy, DefaultEventLoopPolicy):
-asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
-
-loop = None
-try:
-try:
-asyncio.set_child_watcher(None)
-except NotImplementedError:
-pass
-else:
-self.assertTrue(False)
-
-args_tuple = ("hello", "world")
-
-loop = asyncio._wrap_loop()
-future = loop.create_future()
-
-def callback(pid, returncode, *args):
-future.set_result((pid, returncode, args))
-
-async def watch_pid():
-with asyncio.get_child_watcher() as watcher:
-pids = spawn([true_binary], returnpid=True)
-watcher.add_child_handler(pids[0], callback, *args_tuple)
-self.assertEqual((await future), (pids[0], os.EX_OK,
args_tuple))
-
-loop.run_until_complete(watch_pid())
-finally:
-asyncio.set_event_loop_policy(initial_policy)
-if loop not in (None, global_event_loop()):
-loop.close()
-self.assertFalse(global_event_loop().is_closed())
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/tests/ebuild/
commit: 24db094b68a2024cfd23e0366d8a81e8bded5838
Author: Zac Medico gentoo org>
AuthorDate: Wed Oct 4 02:37:00 2023 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Oct 4 02:39:26 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=24db094b
tests: Migrate to ForkProcess target parameter
Bug: https://bugs.gentoo.org/915099
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py| 16 +---
lib/portage/tests/ebuild/test_ipc_daemon.py | 16 +++-
lib/portage/tests/util/futures/test_iter_completed.py | 7 +++
3 files changed, 11 insertions(+), 28 deletions(-)
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index 536e8d8648..51ddc23908 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2016 Gentoo Foundation
+# Copyright 2013-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import portage
@@ -12,13 +12,6 @@ from _emerge.Package import Package
from _emerge.PipeReader import PipeReader
-class DoebuildProcess(ForkProcess):
-__slots__ = ("doebuild_kwargs", "doebuild_pargs")
-
-def _run(self):
-return portage.doebuild(*self.doebuild_pargs, **self.doebuild_kwargs)
-
-
class DoebuildFdPipesTestCase(TestCase):
def testDoebuild(self):
"""
@@ -126,9 +119,10 @@ class DoebuildFdPipesTestCase(TestCase):
):
pr, pw = os.pipe()
-producer = DoebuildProcess(
-doebuild_pargs=(ebuildpath, phase),
-doebuild_kwargs={
+producer = ForkProcess(
+target=portage.doebuild,
+args=(ebuildpath, phase),
+kwargs={
"settings": settings,
"mydbapi": portdb,
"tree": "porttree",
diff --git a/lib/portage/tests/ebuild/test_ipc_daemon.py
b/lib/portage/tests/ebuild/test_ipc_daemon.py
index 55094f9bdd..0beb69ddfc 100644
--- a/lib/portage/tests/ebuild/test_ipc_daemon.py
+++ b/lib/portage/tests/ebuild/test_ipc_daemon.py
@@ -21,18 +21,6 @@ from _emerge.EbuildBuildDir import EbuildBuildDir
from _emerge.EbuildIpcDaemon import EbuildIpcDaemon
-class SleepProcess(ForkProcess):
-"""
-Emulate the sleep command, in order to ensure a consistent
-return code when it is killed by SIGTERM (see bug #437180).
-"""
-
-__slots__ = ("seconds",)
-
-def _run(self):
-time.sleep(self.seconds)
-
-
class IpcDaemonTestCase(TestCase):
_SCHEDULE_TIMEOUT = 40 # seconds
@@ -124,7 +112,9 @@ class IpcDaemonTestCase(TestCase):
daemon = EbuildIpcDaemon(
commands=commands, input_fifo=input_fifo,
output_fifo=output_fifo
)
-proc = SleepProcess(seconds=sleep_time_s)
+# Emulate the sleep command, in order to ensure a consistent
+# return code when it is killed by SIGTERM (see bug #437180).
+proc = ForkProcess(target=time.sleep, args=(sleep_time_s,))
task_scheduler = TaskScheduler(
iter([daemon, proc]), max_jobs=2, event_loop=event_loop
)
diff --git a/lib/portage/tests/util/futures/test_iter_completed.py
b/lib/portage/tests/util/futures/test_iter_completed.py
index bda900505b..0c549018e5 100644
--- a/lib/portage/tests/util/futures/test_iter_completed.py
+++ b/lib/portage/tests/util/futures/test_iter_completed.py
@@ -1,8 +1,9 @@
-# Copyright 2023 Gentoo Foundation
+# Copyright 2018-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import time
+import functools
import pytest
from portage.tests import TestCase
@@ -19,6 +20,7 @@ class SleepProcess(ForkProcess):
__slots__ = ("future", "seconds")
def _start(self):
+self.target = functools.partial(time.sleep, self.seconds)
self.addExitListener(self._future_done)
ForkProcess._start(self)
@@ -26,9 +28,6 @@ class SleepProcess(ForkProcess):
if not self.future.cancelled():
self.future.set_result(self.seconds)
-def _run(self):
-time.sleep(self.seconds)
-
class IterCompletedTestCase(TestCase):
# Mark this as todo, since we don't want to fail if heavy system load
causes
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/asyncio/, lib/portage/util/futures/, lib/portage/, ...
commit: 4c6416780a38e2103fc6e871780ed37b8d60badb
Author: James Le Cuirot gentoo org>
AuthorDate: Fri Jul 21 14:52:56 2023 +
Commit: Sam James gentoo org>
CommitDate: Wed Aug 2 06:31:19 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4c641678
Linting fixes
Signed-off-by: James Le Cuirot gentoo.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/dbapi/__init__.py | 15 ---
lib/portage/dbapi/porttree.py | 13 +++--
.../ebuild/_parallel_manifest/ManifestScheduler.py | 2 +-
lib/portage/sync/modules/cvs/cvs.py | 4 ++--
.../tests/util/futures/asyncio/test_subprocess_exec.py | 4 ++--
lib/portage/util/futures/retry.py | 2 +-
lib/portage/util/socks5.py | 2 +-
lib/portage/versions.py | 17 +
8 files changed, 31 insertions(+), 28 deletions(-)
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 428e4a48e..233da8d69 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -5,7 +5,8 @@ __all__ = ["dbapi"]
import re
import warnings
-from typing import Any, Dict, List, Optional, Sequence, Tuple
+from typing import Any, Dict, List, Optional, Tuple
+from collections.abc import Sequence
import portage
@@ -30,7 +31,7 @@ from _emerge.Package import Package
class dbapi:
_category_re = re.compile(r"^\w[-.+\w]*$", re.UNICODE)
-_categories: Optional[Tuple[str, ...]] = None
+_categories: Optional[tuple[str, ...]] = None
_use_mutable = False
_known_keys = frozenset(auxdbkeys)
_pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository")
@@ -39,7 +40,7 @@ class dbapi:
pass
@property
-def categories(self) -> Tuple[str, ...]:
+def categories(self) -> tuple[str, ...]:
"""
Use self.cp_all() to generate a category list. Mutable instances
can delete the self._categories attribute in cases when the cached
@@ -77,7 +78,7 @@ class dbapi:
# dict to map strings back to their original values.
cpv_list.sort(key=cmp_sort_key(dbapi._cmp_cpv))
-def cpv_all(self) -> List[str]:
+def cpv_all(self) -> list[str]:
"""Return all CPVs in the db
Args:
None
@@ -94,7 +95,7 @@ class dbapi:
cpv_list.extend(self.cp_list(cp))
return cpv_list
-def cp_all(self, sort: bool = False) -> List[str]:
+def cp_all(self, sort: bool = False) -> list[str]:
"""Implement this in a child class
Args
sort - return sorted results
@@ -105,7 +106,7 @@ class dbapi:
def aux_get(
self, mycpv: str, mylist: str, myrepo: Optional[str] = None
-) -> List[str]:
+) -> list[str]:
"""Return the metadata keys in mylist for mycpv
Args:
mycpv - "sys-apps/foo-1.0"
@@ -117,7 +118,7 @@ class dbapi:
"""
raise NotImplementedError
-def aux_update(self, cpv: str, metadata_updates: Dict[str, Any]) -> None:
+def aux_update(self, cpv: str, metadata_updates: dict[str, Any]) -> None:
"""
Args:
cpv - "sys-apps/foo-1.0"
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index c47b66bda..fdfb6ef52 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -49,7 +49,8 @@ import functools
import collections
from collections import OrderedDict
-from typing import List, Optional, Sequence, Type, Tuple, Union
+from collections.abc import Sequence
+from typing import Optional, Union
from urllib.parse import urlparse
@@ -502,7 +503,7 @@ class portdbapi(dbapi):
mycpv: str,
mytree: Optional[str] = None,
myrepo: Optional[str] = None,
-) -> Union[Tuple[None, int], Tuple[str, str], Tuple[str, None]]:
+) -> Union[tuple[None, int], tuple[str, str], tuple[str, None]]:
"""
Returns the location of the CPV, and what overlay it was in.
Searches overlays first, then PORTDIR; this allows us to return the
first
@@ -657,7 +658,7 @@ class portdbapi(dbapi):
mylist: Sequence[str],
mytree: Optional[str] = None,
myrepo: Optional[str] = None,
-) -> List[str]:
+) -> list[str]:
"stub code for returning auxilliary db information, such as SLOT,
DEPEND, etc."
'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com;] or raise
PortageKeyError if error'
@@ -1216,9 +1217,9 @@ class portdbapi(dbapi):
self,
level: str,
origdep: str,
-mydep: Type[DeprecationWarning] = DeprecationWarning,
-mykey: Type[DeprecationWarning] = DeprecationWarning,
-mylist: Type[DeprecationWarning] = DeprecationWarning,
+mydep:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/
commit: 98536f208194197c521675e0d0072bdc599e015a
Author: Zac Medico gentoo org>
AuthorDate: Fri Nov 25 21:21:30 2022 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Nov 25 23:40:38 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=98536f20
testOverallTimeoutWithException: handle TimeoutError
Bug: https://bugs.gentoo.org/850127
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/futures/test_retry.py | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/portage/tests/util/futures/test_retry.py
b/lib/portage/tests/util/futures/test_retry.py
index 8ea832136..cea3e83f5 100644
--- a/lib/portage/tests/util/futures/test_retry.py
+++ b/lib/portage/tests/util/futures/test_retry.py
@@ -176,8 +176,13 @@ class RetryTestCase(TestCase):
asyncio.wait([decorated_func()], loop=loop)
)
self.assertEqual(len(done), 1)
+cause = done.pop().exception().__cause__
self.assertTrue(
-isinstance(done.pop().exception().__cause__,
SucceedNeverException)
+isinstance(
+cause,
+(asyncio.TimeoutError, SucceedNeverException),
+),
+msg=f"Cause was {cause.__class__.__name__}",
)
def testOverallTimeoutWithTimeoutError(self):
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/asyncio/
commit: 447ce6fa4a081ff416431a00a43827f67468b015 Author: Zac Medico gentoo org> AuthorDate: Mon Jan 18 11:56:16 2021 + Commit: Zac Medico gentoo org> CommitDate: Mon Jan 18 11:56:42 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=447ce6fa ChildWatcherTestCase: Use async and await syntax Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/futures/asyncio/test_child_watcher.py | 10 -- 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/portage/tests/util/futures/asyncio/test_child_watcher.py b/lib/portage/tests/util/futures/asyncio/test_child_watcher.py index cd547f008..4e7b4fd7f 100644 --- a/lib/portage/tests/util/futures/asyncio/test_child_watcher.py +++ b/lib/portage/tests/util/futures/asyncio/test_child_watcher.py @@ -1,4 +1,4 @@ -# Copyright 2018-2019 Gentoo Authors +# Copyright 2018-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os @@ -7,7 +7,6 @@ from portage.process import find_binary, spawn from portage.tests import TestCase from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.futures import asyncio -from portage.util.futures.compat_coroutine import coroutine from portage.util.futures.unix_events import DefaultEventLoopPolicy @@ -37,17 +36,16 @@ class ChildWatcherTestCase(TestCase): def callback(pid, returncode, *args): future.set_result((pid, returncode, args)) - @coroutine - def watch_pid(loop=None): + async def watch_pid(): with asyncio.get_child_watcher() as watcher: pids = spawn([true_binary], returnpid=True) watcher.add_child_handler(pids[0], callback, *args_tuple) self.assertEqual( - (yield future), + (await future), (pids[0], os.EX_OK, args_tuple)) - loop.run_until_complete(watch_pid(loop=loop)) + loop.run_until_complete(watch_pid()) finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()):
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/asyncio/
commit: 9e6914ffe118457afbd29f448d99052d90e5e3dd
Author: Zac Medico gentoo org>
AuthorDate: Mon Jan 18 12:04:02 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jan 18 12:04:25 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9e6914ff
SubprocessExecTestCase: Use async and await syntax
Signed-off-by: Zac Medico gentoo.org>
.../util/futures/asyncio/test_subprocess_exec.py | 32 ++
1 file changed, 14 insertions(+), 18 deletions(-)
diff --git a/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
b/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
index 6128a7d06..f9e35f6d4 100644
--- a/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
+++ b/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
@@ -1,4 +1,4 @@
-# Copyright 2018-2019 Gentoo Authors
+# Copyright 2018-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
@@ -9,7 +9,6 @@ from portage.tests import TestCase
from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util.futures import asyncio
from portage.util.futures._asyncio import create_subprocess_exec
-from portage.util.futures.compat_coroutine import coroutine, coroutine_return
from portage.util.futures.unix_events import DefaultEventLoopPolicy
@@ -35,41 +34,38 @@ class SubprocessExecTestCase(TestCase):
echo_binary = echo_binary.encode()
def test(loop):
- @coroutine
- def test_coroutine(loop=None):
- proc = (yield
create_subprocess_exec(echo_binary, *args_tuple,
- stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
- loop=loop))
+ async def test_coroutine():
- out, err = (yield proc.communicate())
+ proc = await
create_subprocess_exec(echo_binary, *args_tuple,
+ stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
+
+ out, err = await proc.communicate()
self.assertEqual(tuple(out.split()), args_tuple)
self.assertEqual(proc.returncode, os.EX_OK)
- proc = (yield create_subprocess_exec(
+ proc = await create_subprocess_exec(
'bash', '-c', 'echo foo; echo
bar 1>&2;',
- stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
- loop=loop))
+ stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
- out, err = (yield proc.communicate())
+ out, err = await proc.communicate()
self.assertEqual(out, b'foo\n')
self.assertEqual(err, b'bar\n')
self.assertEqual(proc.returncode, os.EX_OK)
- proc = (yield create_subprocess_exec(
+ proc = await create_subprocess_exec(
'bash', '-c', 'echo foo; echo
bar 1>&2;',
- stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
- loop=loop))
+ stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
- out, err = (yield proc.communicate())
+ out, err = await proc.communicate()
self.assertEqual(out, b'foo\nbar\n')
self.assertEqual(err, None)
self.assertEqual(proc.returncode, os.EX_OK)
- coroutine_return('success')
+ return 'success'
self.assertEqual('success',
-
loop.run_until_complete(test_coroutine(loop=loop)))
+ loop.run_until_complete(test_coroutine()))
self._run_test(test)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/
commit: be0b5d33ee8b355dcc9932b52bb1e025e6bd58d4
Author: Zac Medico gentoo org>
AuthorDate: Mon Jan 18 11:16:53 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jan 18 11:31:57 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=be0b5d33
RetryTestCase: Use async and await syntax
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/futures/test_retry.py | 28 +---
1 file changed, 9 insertions(+), 19 deletions(-)
diff --git a/lib/portage/tests/util/futures/test_retry.py
b/lib/portage/tests/util/futures/test_retry.py
index 6648b1b2c..bce48a693 100644
--- a/lib/portage/tests/util/futures/test_retry.py
+++ b/lib/portage/tests/util/futures/test_retry.py
@@ -1,4 +1,4 @@
-# Copyright 2018-2020 Gentoo Authors
+# Copyright 2018-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from concurrent.futures import Future, ThreadPoolExecutor
@@ -32,18 +32,11 @@ class SucceedLater:
def __init__(self, duration):
self._succeed_time = time.monotonic() + duration
- def __call__(self):
- loop = global_event_loop()
- result = loop.create_future()
+ async def __call__(self):
remaining = self._succeed_time - time.monotonic()
if remaining > 0:
- loop.call_soon_threadsafe(lambda: None if result.done()
else
- result.set_exception(SucceedLaterException(
- 'time until success: {}
seconds'.format(remaining
- else:
- loop.call_soon_threadsafe(lambda: None if result.done()
else
- result.set_result('success'))
- return result
+ await asyncio.sleep(remaining)
+ return 'success'
class SucceedNeverException(Exception):
@@ -54,20 +47,17 @@ class SucceedNever:
"""
A callable object that never succeeds.
"""
- def __call__(self):
- loop = global_event_loop()
- result = loop.create_future()
- loop.call_soon_threadsafe(lambda: None if result.done() else
- result.set_exception(SucceedNeverException('expected
failure')))
- return result
+ async def __call__(self):
+ raise SucceedNeverException('expected failure')
class HangForever:
"""
A callable object that sleeps forever.
"""
- def __call__(self):
- return asyncio.Future()
+ async def __call__(self):
+ while True:
+ await asyncio.sleep(9)
class RetryTestCase(TestCase):
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/
commit: e93549105d5f009e47710db93abea9a7aeb34324 Author: Zac Medico gentoo org> AuthorDate: Mon Jan 4 07:38:55 2021 + Commit: Zac Medico gentoo org> CommitDate: Mon Jan 4 08:11:51 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e9354910 test_retry: use context manager to cancel pending futures Cancel pending futures in order to avoid this "Task was destroyed but it is pending!" warning message following migration to PEP 492 coroutines with async and await syntax: testHangForever (portage.tests.util.futures.test_retry.RetryForkExecutorTestCase) ... ok testHangForever (portage.tests.util.futures.test_retry.RetryTestCase) ... ok testHangForever (portage.tests.util.futures.test_retry.RetryThreadExecutorTestCase) ... ok -- Ran 3 tests in 0.839s OK Task was destroyed but it is pending! task: wait_for= cb=[RetryForkExecutorTestCase._wrap_coroutine_func..wrapper..done_callback() at portage/tests/util/futures/test_retry.py:192]> Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/futures/test_retry.py | 181 --- 1 file changed, 106 insertions(+), 75 deletions(-) diff --git a/lib/portage/tests/util/futures/test_retry.py b/lib/portage/tests/util/futures/test_retry.py index ce5fb3e11..6648b1b2c 100644 --- a/lib/portage/tests/util/futures/test_retry.py +++ b/lib/portage/tests/util/futures/test_retry.py @@ -1,15 +1,18 @@ # Copyright 2018-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, ThreadPoolExecutor +import contextlib try: import threading except ImportError: import dummy_threading as threading +import weakref import time +import portage from portage.tests import TestCase from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.backoff import RandomExponentialBackoff @@ -64,99 +67,100 @@ class HangForever: A callable object that sleeps forever. """ def __call__(self): - return global_event_loop().create_future() + return asyncio.Future() class RetryTestCase(TestCase): + @contextlib.contextmanager def _wrap_coroutine_func(self, coroutine_func): """ Derived classes may override this method in order to implement alternative forms of execution. """ - return coroutine_func + yield coroutine_func def testSucceedLater(self): loop = global_event_loop() - func_coroutine = self._wrap_coroutine_func(SucceedLater(1)) - decorator = retry(try_max=, - delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine, loop=loop) - result = loop.run_until_complete(decorated_func()) - self.assertEqual(result, 'success') + with self._wrap_coroutine_func(SucceedLater(1)) as func_coroutine: + decorator = retry(try_max=, + delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) + decorated_func = decorator(func_coroutine, loop=loop) + result = loop.run_until_complete(decorated_func()) + self.assertEqual(result, 'success') def testSucceedNever(self): loop = global_event_loop() - func_coroutine = self._wrap_coroutine_func(SucceedNever()) - decorator = retry(try_max=4, try_timeout=None, - delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) - decorated_func = decorator(func_coroutine, loop=loop) - done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) - self.assertEqual(len(done), 1) - self.assertTrue(isinstance(done.pop().exception().__cause__, SucceedNeverException)) + with self._wrap_coroutine_func(SucceedNever()) as func_coroutine: + decorator = retry(try_max=4, try_timeout=None, + delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) + decorated_func = decorator(func_coroutine, loop=loop) + done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) + self.assertEqual(len(done), 1) + self.assertTrue(isinstance(done.pop().exception().__cause__, SucceedNeverException)) def testSucceedNeverReraise(self): loop = global_event_loop() - func_coroutine = self._wrap_coroutine_func(SucceedNever()) - decorator =
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/asyncio/
commit: 570faa2a3bd095e4ec86abcf85df3c79a0b418cf Author: Zac Medico gentoo org> AuthorDate: Sat Aug 8 03:01:35 2020 + Commit: Zac Medico gentoo org> CommitDate: Sat Aug 8 03:03:02 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=570faa2a EventLoopInForkTestCase: use AsyncFunction Signed-off-by: Zac Medico gentoo.org> .../futures/asyncio/test_event_loop_in_fork.py | 23 +- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/lib/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py b/lib/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py index 177953437..e409fd52b 100644 --- a/lib/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py +++ b/lib/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py @@ -1,17 +1,16 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import multiprocessing import os from portage.tests import TestCase +from portage.util._async.AsyncFunction import AsyncFunction from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.futures import asyncio from portage.util.futures.unix_events import DefaultEventLoopPolicy -def fork_main(parent_conn, child_conn): - parent_conn.close() +def fork_main(): loop = asyncio._wrap_loop() # This fails with python's default event loop policy, # see https://bugs.python.org/issue22087. @@ -21,21 +20,9 @@ def fork_main(parent_conn, child_conn): def async_main(fork_exitcode, loop=None): loop = asyncio._wrap_loop(loop) - - # Since python2.7 does not support Process.sentinel, use Pipe to - # monitor for process exit. - parent_conn, child_conn = multiprocessing.Pipe() - - def eof_callback(proc): - loop.remove_reader(parent_conn.fileno()) - parent_conn.close() - proc.join() - fork_exitcode.set_result(proc.exitcode) - - proc = multiprocessing.Process(target=fork_main, args=(parent_conn, child_conn)) - loop.add_reader(parent_conn.fileno(), eof_callback, proc) + proc = AsyncFunction(scheduler=loop, target=fork_main) proc.start() - child_conn.close() + proc.async_wait().add_done_callback(lambda future: fork_exitcode.set_result(future.result())) class EventLoopInForkTestCase(TestCase):
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/, lib/portage/util/_dyn_libs/, ...
commit: 27378ccf8e497f6d5d938470db8a9caacba176ab Author: Michał Górny gentoo org> AuthorDate: Thu Jul 16 19:04:44 2020 + Commit: Michał Górny gentoo org> CommitDate: Fri Jul 17 04:27:19 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=27378ccf Clean up more py2 conditional code Closes: https://github.com/gentoo/portage/pull/575 Reviewed-by: Zac Medico gentoo.org> Signed-off-by: Michał Górny gentoo.org> lib/portage/cache/anydbm.py| 4 +- lib/portage/cache/mappings.py | 45 ++ lib/portage/cache/sql_template.py | 5 +- lib/portage/cache/template.py | 12 ++--- lib/portage/elog/messages.py | 3 +- lib/portage/output.py | 3 +- lib/portage/package/ebuild/config.py | 11 + .../util/futures/asyncio/test_subprocess_exec.py | 4 -- lib/portage/tests/util/futures/test_retry.py | 2 - lib/portage/tests/util/test_socks5.py | 16 ++- lib/portage/util/__init__.py | 53 +- lib/portage/util/_dyn_libs/NeededEntry.py | 10 lib/portage/util/digraph.py| 3 -- lib/portage/util/listdir.py| 2 - lib/portage/util/whirlpool.py | 25 +- lib/portage/xpak.py| 2 - 16 files changed, 52 insertions(+), 148 deletions(-) diff --git a/lib/portage/cache/anydbm.py b/lib/portage/cache/anydbm.py index 88d85b0da..121a4eaf2 100644 --- a/lib/portage/cache/anydbm.py +++ b/lib/portage/cache/anydbm.py @@ -112,5 +112,5 @@ class database(fs_template.FsBased): self.__db.sync() self.__db.close() - if sys.hexversion >= 0x300: - items = iteritems + # TODO: do we need iteritems()? + items = iteritems diff --git a/lib/portage/cache/mappings.py b/lib/portage/cache/mappings.py index 0432fdf60..0adecde4a 100644 --- a/lib/portage/cache/mappings.py +++ b/lib/portage/cache/mappings.py @@ -25,9 +25,6 @@ class Mapping(object): def __iter__(self): return iter(self.keys()) - def keys(self): - return list(self.__iter__()) - def __contains__(self, key): try: value = self[key] @@ -46,12 +43,6 @@ class Mapping(object): for _, v in self.items(): yield v - def values(self): - return [v for _, v in self.iteritems()] - - def items(self): - return list(self.iteritems()) - def get(self, key, default=None): try: return self[key] @@ -64,10 +55,10 @@ class Mapping(object): def __len__(self): return len(list(self)) - if sys.hexversion >= 0x300: - items = iteritems - keys = __iter__ - values = itervalues + # TODO: do we need to keep iter*? + items = iteritems + keys = __iter__ + values = itervalues class MutableMapping(Mapping): """ @@ -184,8 +175,8 @@ class UserDict(MutableMapping): def clear(self): self.data.clear() - if sys.hexversion >= 0x300: - keys = __iter__ + keys = __iter__ + class ProtectedDict(MutableMapping): """ @@ -234,8 +225,8 @@ class ProtectedDict(MutableMapping): def __contains__(self, key): return key in self.new or (key not in self.blacklist and key in self.orig) - if sys.hexversion >= 0x300: - keys = __iter__ + keys = __iter__ + class LazyLoad(Mapping): """ @@ -271,8 +262,8 @@ class LazyLoad(Mapping): self.pull = None return key in self.d - if sys.hexversion >= 0x300: - keys = __iter__ + keys = __iter__ + _slot_dict_classes = weakref.WeakValueDictionary() @@ -328,9 +319,6 @@ def slot_dict_class(keys, prefix="_val_"): l += 1 return l - def keys(self): - return list(self) - def iteritems(self): prefix = self._prefix for k in self.allowed_keys: @@ -339,16 +327,10 @@ def slot_dict_class(keys, prefix="_val_"): except AttributeError: pass - def items(self): - return list(self.iteritems()) - def itervalues(self): for k, v in self.iteritems(): yield v - def
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/, ...
commit: bf1505777847aface5b4cac16b955071c2915ddd
Author: Zac Medico gentoo org>
AuthorDate: Wed Apr 8 05:03:34 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Apr 8 05:29:48 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bf150577
Revert "AsyncScheduler: use async_start method"
This reverts commit 8f47d3fe1190d4476ae9eebfafcebdfb1794fc05.
Bug: https://bugs.gentoo.org/716636
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 8 +++--
.../tests/util/futures/test_iter_completed.py | 2 --
lib/portage/util/_async/AsyncScheduler.py | 20 ++--
lib/portage/util/futures/iter_completed.py | 38 +-
4 files changed, 15 insertions(+), 53 deletions(-)
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index 50fc5fe1c..05ea24c4b 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -109,16 +109,18 @@ class DoebuildFdPipesTestCase(TestCase):
output_fd: pw,
},
"prev_mtimes": {}})
- producer.addStartListener(lambda producer:
os.close(pw))
- # PipeReader closes pr
consumer = PipeReader(
input_files={"producer" : pr})
task_scheduler = TaskScheduler(iter([producer,
consumer]),
max_jobs=2)
-
loop.run_until_complete(task_scheduler.async_start())
+ try:
+
loop.run_until_complete(task_scheduler.async_start())
+ finally:
+ # PipeReader closes pr
+ os.close(pw)
task_scheduler.wait()
output = portage._unicode_decode(
diff --git a/lib/portage/tests/util/futures/test_iter_completed.py
b/lib/portage/tests/util/futures/test_iter_completed.py
index 03ace915a..aa24f5685 100644
--- a/lib/portage/tests/util/futures/test_iter_completed.py
+++ b/lib/portage/tests/util/futures/test_iter_completed.py
@@ -76,8 +76,6 @@ class IterCompletedTestCase(TestCase):
for future_done_set in async_iter_completed(future_generator(),
max_jobs=True, max_load=True, loop=loop):
- while not input_futures:
- loop.run_until_complete(asyncio.sleep(0,
loop=loop))
future_done_set.cancel()
break
diff --git a/lib/portage/util/_async/AsyncScheduler.py
b/lib/portage/util/_async/AsyncScheduler.py
index b9070061a..c6b523eaa 100644
--- a/lib/portage/util/_async/AsyncScheduler.py
+++ b/lib/portage/util/_async/AsyncScheduler.py
@@ -1,11 +1,7 @@
# Copyright 2012-2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-import functools
-
from portage import os
-from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import coroutine
from _emerge.AsynchronousTask import AsynchronousTask
from _emerge.PollScheduler import PollScheduler
@@ -66,8 +62,8 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
else:
self._running_tasks.add(task)
task.scheduler = self._sched_iface
- future =
asyncio.ensure_future(self._task_coroutine(task), loop=self._sched_iface)
-
future.add_done_callback(functools.partial(self._task_coroutine_done, task))
+ task.addExitListener(self._task_exit)
+ task.start()
if self._loadavg_check_id is not None:
self._loadavg_check_id.cancel()
@@ -77,18 +73,6 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
# Triggers cleanup and exit listeners if there's nothing left
to do.
self.poll()
- @coroutine
- def _task_coroutine(self, task):
- yield task.async_start()
- yield task.async_wait()
-
- def _task_coroutine_done(self, task, future):
- try:
- future.result()
- except asyncio.CancelledError:
- self.cancel()
- self._task_exit(task)
-
def _task_exit(self, task):
self._running_tasks.discard(task)
if task.returncode != os.EX_OK:
diff --git
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/_emerge/
commit: 5d476c4e500248929d6b042de302b1e7c923dc60 Author: Zac Medico gentoo org> AuthorDate: Fri Mar 6 08:11:05 2020 + Commit: Zac Medico gentoo org> CommitDate: Fri Mar 6 08:55:17 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5d476c4e AsynchronousTask: handle addExistListener after exit When addExistListener is called after the task has already exited with a returncode, immediately schedule the listener to be invoked via call_soon. This behavior is similar to the Future add_done_callback method. Signed-off-by: Zac Medico gentoo.org> lib/_emerge/AsynchronousTask.py| 2 ++ .../util/futures/test_done_callback_after_exit.py | 40 ++ 2 files changed, 42 insertions(+) diff --git a/lib/_emerge/AsynchronousTask.py b/lib/_emerge/AsynchronousTask.py index 799e66a4a..97db02587 100644 --- a/lib/_emerge/AsynchronousTask.py +++ b/lib/_emerge/AsynchronousTask.py @@ -176,6 +176,8 @@ class AsynchronousTask(SlotObject): if self._exit_listeners is None: self._exit_listeners = [] self._exit_listeners.append(f) + if self.returncode is not None: + self._wait_hook() def removeExitListener(self, f): if self._exit_listeners is not None: diff --git a/lib/portage/tests/util/futures/test_done_callback_after_exit.py b/lib/portage/tests/util/futures/test_done_callback_after_exit.py new file mode 100644 index 0..46a51c271 --- /dev/null +++ b/lib/portage/tests/util/futures/test_done_callback_after_exit.py @@ -0,0 +1,40 @@ +# Copyright 2020 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from _emerge.AsynchronousTask import AsynchronousTask +from portage.tests import TestCase +from portage.util.futures import asyncio + + +class DoneCallbackAfterExitTestCase(TestCase): + + def test_done_callback_after_exit(self): + """ + Test that callbacks can be registered via the Future + add_done_callback method even after the future is done, and + verify that the callbacks are called. + """ + loop = asyncio._wrap_loop() + future = loop.create_future() + future.set_result(None) + + for i in range(3): + event = loop.create_future() + future.add_done_callback(lambda future: event.set_result(None)) + loop.run_until_complete(event) + + def test_exit_listener_after_exit(self): + """ + Test that callbacks can be registered via the AsynchronousTask + addExitListener method even after the task is done, and + verify that the callbacks are called. + """ + loop = asyncio._wrap_loop() + task = AsynchronousTask(scheduler=loop) + loop.run_until_complete(task.async_start()) + loop.run_until_complete(task.async_wait()) + + for i in range(3): + event = loop.create_future() + task.addExitListener(lambda task: event.set_result(None)) + loop.run_until_complete(event)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/, lib/_emerge/
commit: bab11fcee344df488d2e7f444ea3711ce87669e3
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 1 21:56:41 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Mar 2 00:35:51 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bab11fce
_GeneratorTask: throw CancelledError in cancelled coroutine (bug 711174)
Throw asyncio.CancelledError in a cancelled coroutine, ensuring
that the coroutine can handle this exception in order to perform
any necessary cleanup (like close the log file for bug 711174).
Note that the asyncio.CancelledError will only be thrown in the
coroutine if there's an opportunity (yield) before the generator
raises StopIteration.
Also fix the AsynchronousTask exit listener handling for
compatibility with this new behavior.
Fixes: 8074127bbc21 ("SpawnProcess: add _main coroutine")
Bug: https://bugs.gentoo.org/711174
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/AsynchronousTask.py| 12 ++---
.../tests/util/futures/test_compat_coroutine.py| 29 +++---
lib/portage/util/futures/compat_coroutine.py | 19 ++
3 files changed, 49 insertions(+), 11 deletions(-)
diff --git a/lib/_emerge/AsynchronousTask.py b/lib/_emerge/AsynchronousTask.py
index 1e9e177cb..580eef050 100644
--- a/lib/_emerge/AsynchronousTask.py
+++ b/lib/_emerge/AsynchronousTask.py
@@ -64,7 +64,7 @@ class AsynchronousTask(SlotObject):
@returns: Future, result is self.returncode
"""
waiter = self.scheduler.create_future()
- exit_listener = lambda self: waiter.set_result(self.returncode)
+ exit_listener = lambda self: waiter.cancelled() or
waiter.set_result(self.returncode)
self.addExitListener(exit_listener)
waiter.add_done_callback(lambda waiter:
self.removeExitListener(exit_listener) if
waiter.cancelled() else None)
@@ -180,9 +180,15 @@ class AsynchronousTask(SlotObject):
def removeExitListener(self, f):
if self._exit_listeners is None:
if self._exit_listener_stack is not None:
- self._exit_listener_stack.remove(f)
+ try:
+ self._exit_listener_stack.remove(f)
+ except ValueError:
+ pass
return
- self._exit_listeners.remove(f)
+ try:
+ self._exit_listeners.remove(f)
+ except ValueError:
+ pass
def _wait_hook(self):
"""
diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py
b/lib/portage/tests/util/futures/test_compat_coroutine.py
index f96aa9be5..b561c0227 100644
--- a/lib/portage/tests/util/futures/test_compat_coroutine.py
+++ b/lib/portage/tests/util/futures/test_compat_coroutine.py
@@ -57,20 +57,43 @@ class CompatCoroutineTestCase(TestCase):
loop.run_until_complete(catching_coroutine(loop=loop)))
def test_cancelled_coroutine(self):
+ """
+ Verify that a coroutine can handle (and reraise)
asyncio.CancelledError
+ in order to perform any necessary cleanup. Note that the
+ asyncio.CancelledError will only be thrown in the coroutine if
there's
+ an opportunity (yield) before the generator raises
StopIteration.
+ """
+ loop = asyncio.get_event_loop()
+ ready_for_exception = loop.create_future()
+ exception_in_coroutine = loop.create_future()
@coroutine
def cancelled_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
while True:
- yield loop.create_future()
+ task = loop.create_future()
+ try:
+ ready_for_exception.set_result(None)
+ yield task
+ except BaseException as e:
+ # Since python3.8,
asyncio.CancelledError inherits
+ # from BaseException.
+ task.done() or task.cancel()
+ exception_in_coroutine.set_exception(e)
+ raise
+ else:
+ exception_in_coroutine.set_result(None)
- loop = asyncio.get_event_loop()
future = cancelled_coroutine(loop=loop)
- loop.call_soon(future.cancel)
+ loop.run_until_complete(ready_for_exception)
+ future.cancel()
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/dbapi/, lib/portage/util/_async/, ...
commit: 5c40c3e7ec180c9c7d1eea521d69487177c7f519
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 23 23:17:16 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 24 02:35:15 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5c40c3e7
SpawnProcess: use async_start method (bug 709746)
Convert SpawnProcess to use the async_start method, since
eventually this method will need to be a coroutine in order to write
messages to the build log as discussed in bug 709746.
Also update AbstractEbuildProcess, AsynchronousLock,
BinpkgExtractorAsync, BinpkgFetcher, EbuildFetcher,
IterCompletedTestCase, AsyncFunction, and FileDigester for
compatibility with SpawnProcess async_start.
Bug: https://bugs.gentoo.org/709746
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/AbstractEbuildProcess.py | 2 +-
lib/_emerge/AsynchronousLock.py | 15 ---
lib/_emerge/BinpkgExtractorAsync.py | 9 +++--
lib/_emerge/BinpkgFetcher.py | 9 +++--
lib/_emerge/EbuildFetcher.py | 9 +++--
lib/_emerge/SpawnProcess.py | 8 ++--
lib/portage/dbapi/bintree.py | 4 ++--
lib/portage/tests/util/futures/test_iter_completed.py | 6 +-
lib/portage/util/_async/AsyncFunction.py | 9 +++--
lib/portage/util/_async/FileDigester.py | 9 +++--
10 files changed, 61 insertions(+), 19 deletions(-)
diff --git a/lib/_emerge/AbstractEbuildProcess.py
b/lib/_emerge/AbstractEbuildProcess.py
index 7eb5dfd1b..d1a6d1c4e 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -182,7 +182,7 @@ class AbstractEbuildProcess(SpawnProcess):
self.fd_pipes[0] = null_fd
try:
- SpawnProcess._start(self)
+ yield SpawnProcess._async_start(self)
finally:
if null_fd is not None:
os.close(null_fd)
diff --git a/lib/_emerge/AsynchronousLock.py b/lib/_emerge/AsynchronousLock.py
index aed1bcb15..9efaaceac 100644
--- a/lib/_emerge/AsynchronousLock.py
+++ b/lib/_emerge/AsynchronousLock.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2018 Gentoo Foundation
+# Copyright 2010-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import fcntl
@@ -21,6 +21,7 @@ from portage.exception import TryAgain
from portage.localization import _
from portage.locks import lockfile, unlockfile
from portage.util import writemsg_level
+from portage.util.futures.compat_coroutine import coroutine
from _emerge.AbstractPollTask import AbstractPollTask
from _emerge.AsynchronousTask import AsynchronousTask
from _emerge.SpawnProcess import SpawnProcess
@@ -43,6 +44,10 @@ class AsynchronousLock(AsynchronousTask):
_use_process_by_default = True
def _start(self):
+ self.scheduler.run_until_complete(self._async_start())
+
+ @coroutine
+ def _async_start(self):
if not self._force_async:
try:
@@ -65,7 +70,7 @@ class AsynchronousLock(AsynchronousTask):
_force_dummy=self._force_dummy)
self._imp.addExitListener(self._imp_exit)
- self._imp.start()
+ yield self._imp.async_start()
def _imp_exit(self, imp):
# call exit listeners
@@ -183,6 +188,10 @@ class _LockProcess(AbstractPollTask):
('_acquired', '_kill_test', '_proc', '_files', '_unlock_future')
def _start(self):
+ self.scheduler.run_until_complete(self._async_start())
+
+ @coroutine
+ def _async_start(self):
in_pr, in_pw = os.pipe()
out_pr, out_pw = os.pipe()
self._files = {}
@@ -211,7 +220,7 @@ class _LockProcess(AbstractPollTask):
fd_pipes={0:out_pr, 1:in_pw,
2:sys.__stderr__.fileno()},
scheduler=self.scheduler)
self._proc.addExitListener(self._proc_exit)
- self._proc.start()
+ yield self._proc.async_start()
os.close(out_pr)
os.close(in_pw)
diff --git a/lib/_emerge/BinpkgExtractorAsync.py
b/lib/_emerge/BinpkgExtractorAsync.py
index 3733bdeb5..5f4caa794 100644
--- a/lib/_emerge/BinpkgExtractorAsync.py
+++ b/lib/_emerge/BinpkgExtractorAsync.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2013 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import logging
@@ -10,6 +10,7 @@ from portage.util.compression_probe import (
compression_probe,
_compressors,
)
+from portage.util.futures.compat_coroutine import coroutine
from portage.process import find_binary
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/tests/locks/, ...
commit: d66e9ec0b10522528d62e18b83e012c1ec121787 Author: Zac Medico gentoo org> AuthorDate: Mon Feb 17 18:32:06 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Feb 17 22:29:09 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d66e9ec0 AsynchronousTask: add coroutine async_start method Add coroutine async_start coroutine method which calls an _async_start template method. Eventually, subclasses having _start implementations that need to write to a build log will be required to implement an _async_start coroutine method to replace the _start method as discussed in bug 709746. Bug: https://bugs.gentoo.org/709746 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/AsynchronousTask.py| 22 +- lib/_emerge/CompositeTask.py | 17 ++--- lib/_emerge/TaskSequence.py| 7 ++- lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 6 -- lib/portage/tests/ebuild/test_doebuild_spawn.py| 6 +++--- lib/portage/tests/ebuild/test_fetch.py | 9 ++--- lib/portage/tests/ebuild/test_ipc_daemon.py| 4 ++-- lib/portage/tests/ebuild/test_spawn.py | 4 ++-- .../test_lazy_import_portage_baseline.py | 6 +++--- lib/portage/tests/locks/test_asynchronous_lock.py | 18 +- lib/portage/tests/process/test_PopenProcess.py | 6 +++--- .../tests/process/test_PopenProcessBlockingIO.py | 4 ++-- lib/portage/tests/process/test_poll.py | 4 ++-- .../tests/util/futures/test_iter_completed.py | 16 +++- lib/portage/tests/util/test_file_copier.py | 6 +++--- 15 files changed, 87 insertions(+), 48 deletions(-) diff --git a/lib/_emerge/AsynchronousTask.py b/lib/_emerge/AsynchronousTask.py index cf6e6dc44..280ed16da 100644 --- a/lib/_emerge/AsynchronousTask.py +++ b/lib/_emerge/AsynchronousTask.py @@ -1,10 +1,11 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import signal from portage import os from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine, coroutine_return from portage.util.SlotObject import SlotObject class AsynchronousTask(SlotObject): @@ -22,6 +23,17 @@ class AsynchronousTask(SlotObject): _cancelled_returncode = - signal.SIGINT + @coroutine + def async_start(self): + self._start_hook() + yield self._async_start() + + @coroutine + def _async_start(self): + self._start() + coroutine_return() + yield None + def start(self): """ Start an asynchronous task and then return as soon as possible. @@ -29,6 +41,10 @@ class AsynchronousTask(SlotObject): self._start_hook() self._start() + def _start(self): + self.returncode = os.EX_OK + self._async_wait() + def async_wait(self): """ Wait for returncode asynchronously. Notification is available @@ -49,10 +65,6 @@ class AsynchronousTask(SlotObject): self._async_wait() return waiter - def _start(self): - self.returncode = os.EX_OK - self._async_wait() - def isAlive(self): return self.returncode is None diff --git a/lib/_emerge/CompositeTask.py b/lib/_emerge/CompositeTask.py index 1edec4a17..ce7136c3d 100644 --- a/lib/_emerge/CompositeTask.py +++ b/lib/_emerge/CompositeTask.py @@ -1,8 +1,10 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from _emerge.AsynchronousTask import AsynchronousTask from portage import os +from portage.util.futures import asyncio + class CompositeTask(AsynchronousTask): @@ -97,7 +99,7 @@ class CompositeTask(AsynchronousTask): def _start_task(self, task, exit_handler): """ Register exit handler for the given task, set it - as self._current_task, and call task.start(). + as self._current_task, and call task.async_start(). Subclasses can use this as a generic way to start a task. @@ -109,7 +111,16 @@ class CompositeTask(AsynchronousTask): pass task.addExitListener(exit_handler) self._current_task = task - task.start() + result = asyncio.ensure_future(task.async_start(), loop=self.scheduler) + result.add_done_callback(self._current_task_start_cb) + + def _current_task_start_cb(self, future): + try: + future.result() + except
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/asyncio/
commit: 40306beadf35659ce6dbe40ffe10c677e6e13918
Author: Zac Medico gentoo org>
AuthorDate: Sat Nov 16 08:52:51 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Nov 16 08:54:49 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=40306bea
SubprocessExecTestCase: test pipe between processes
Signed-off-by: Zac Medico gentoo.org>
.../util/futures/asyncio/test_subprocess_exec.py | 36 --
1 file changed, 34 insertions(+), 2 deletions(-)
diff --git a/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
b/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
index 61646cb92..d7e94d132 100644
--- a/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
+++ b/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
@@ -10,7 +10,6 @@ from portage.tests import TestCase
from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util.futures import asyncio
from portage.util.futures._asyncio import create_subprocess_exec
-from portage.util.futures._asyncio.streams import _reader as reader
from portage.util.futures.compat_coroutine import coroutine, coroutine_return
from portage.util.futures.unix_events import DefaultEventLoopPolicy
@@ -94,6 +93,39 @@ class SubprocessExecTestCase(TestCase):
self._run_test(test)
+ def testPipe(self):
+ stdin_data = b'hello world'
+ cat_binary = find_binary("cat")
+ self.assertNotEqual(cat_binary, None)
+ cat_binary = cat_binary.encode()
+
+ echo_binary = find_binary("echo")
+ self.assertNotEqual(echo_binary, None)
+ echo_binary = echo_binary.encode()
+
+ def test(loop):
+
+ pr, pw = os.pipe()
+
+ cat_proc =
loop.run_until_complete(create_subprocess_exec(
+ cat_binary, stdin=pr, stdout=subprocess.PIPE,
+ loop=loop))
+
+ echo_proc =
loop.run_until_complete(create_subprocess_exec(
+ echo_binary, b'-n', stdin_data, stdout=pw,
+ loop=loop))
+
+ os.close(pr)
+ os.close(pw)
+
+ out, err =
loop.run_until_complete(cat_proc.communicate())
+
+
self.assertEqual(loop.run_until_complete(cat_proc.wait()), os.EX_OK)
+
self.assertEqual(loop.run_until_complete(echo_proc.wait()), os.EX_OK)
+ self.assertEqual(out, stdin_data)
+
+ self._run_test(test)
+
def testReadTransport(self):
"""
Test asyncio.create_subprocess_exec(stdout=subprocess.PIPE)
which
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/asyncio/
commit: e57cc89e8870beff73a65ab4a59fea2fae3c262e Author: Zac Medico gentoo org> AuthorDate: Tue Oct 15 00:40:55 2019 + Commit: Zac Medico gentoo org> CommitDate: Tue Oct 15 00:45:30 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e57cc89e ChildWatcherTestCase: python3.8 compat Since python3.8, the add_child_handler method must be called while the event loop is running, in order to avoid this error: File "lib/portage/tests/util/futures/asyncio/test_child_watcher.py", line 41, in testChildWatcher watcher.add_child_handler(pids[0], callback, *args_tuple) File "/opt/python/3.8-dev/lib/python3.8/asyncio/unix_events.py", line 1286, in add_child_handler loop = events.get_running_loop() RuntimeError: no running event loop See: https://travis-ci.org/gentoo/portage/jobs/597907096 Signed-off-by: Zac Medico gentoo.org> .../tests/util/futures/asyncio/test_child_watcher.py | 19 --- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/portage/tests/util/futures/asyncio/test_child_watcher.py b/lib/portage/tests/util/futures/asyncio/test_child_watcher.py index 0fc73ab49..8a8fb3d4f 100644 --- a/lib/portage/tests/util/futures/asyncio/test_child_watcher.py +++ b/lib/portage/tests/util/futures/asyncio/test_child_watcher.py @@ -1,4 +1,4 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os @@ -7,6 +7,7 @@ from portage.process import find_binary, spawn from portage.tests import TestCase from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine from portage.util.futures.unix_events import DefaultEventLoopPolicy @@ -36,13 +37,17 @@ class ChildWatcherTestCase(TestCase): def callback(pid, returncode, *args): future.set_result((pid, returncode, args)) - with asyncio.get_child_watcher() as watcher: - pids = spawn([true_binary], returnpid=True) - watcher.add_child_handler(pids[0], callback, *args_tuple) + @coroutine + def watch_pid(): - self.assertEqual( - loop.run_until_complete(future), - (pids[0], os.EX_OK, args_tuple)) + with asyncio.get_child_watcher() as watcher: + pids = spawn([true_binary], returnpid=True) + watcher.add_child_handler(pids[0], callback, *args_tuple) + self.assertEqual( + (yield future), + (pids[0], os.EX_OK, args_tuple)) + + loop.run_until_complete(watch_pid()) finally: asyncio.set_event_loop_policy(initial_policy) if loop not in (None, global_event_loop()):
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/asyncio/
commit: 6055796f5c5a4d69b759b0af376e9388aa5767a7
Author: Zac Medico gentoo org>
AuthorDate: Mon Apr 15 23:13:37 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Apr 15 23:15:07 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6055796f
WakeupFdSigchldTestCase: enable test with python2.7
Signed-off-by: Zac Medico gentoo.org>
.../tests/util/futures/asyncio/test_wakeup_fd_sigchld.py | 10 +++---
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
index abc67c241..e5b104e0f 100644
--- a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
+++ b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
@@ -7,7 +7,6 @@ import subprocess
import portage
from portage.const import PORTAGE_PYM_PATH
from portage.tests import TestCase
-from portage.util._eventloop.global_event_loop import _asyncio_enabled
class WakeupFdSigchldTestCase(TestCase):
@@ -19,11 +18,8 @@ class WakeupFdSigchldTestCase(TestCase):
Exception ignored when trying to write to the signal wakeup fd:
BlockingIOError: [Errno 11] Resource temporarily unavailable
"""
- if not _asyncio_enabled:
- self.skipTest('asyncio not enabled')
script = """
-import asyncio as _real_asyncio
import os
import signal
import sys
@@ -39,7 +35,7 @@ from portage.util.futures import asyncio
loop = asyncio._wrap_loop()
# Cause the loop to register a child watcher.
-proc = loop.run_until_complete(_real_asyncio.create_subprocess_exec('sleep',
'0'))
+proc = loop.run_until_complete(asyncio.create_subprocess_exec('sleep', '0',
loop=loop))
loop.run_until_complete(proc.wait())
for i in range(8192):
@@ -47,7 +43,7 @@ for i in range(8192):
# Verify that the child watcher still works correctly
# (this will hang if it doesn't).
-proc = loop.run_until_complete(_real_asyncio.create_subprocess_exec('sleep',
'0'))
+proc = loop.run_until_complete(asyncio.create_subprocess_exec('sleep', '0',
loop=loop))
loop.run_until_complete(proc.wait())
loop.close()
sys.stdout.write('success')
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/
commit: de0b60ff277311e780102131dce3111b4db1c196
Author: Zac Medico gentoo org>
AuthorDate: Sat Jul 28 13:53:11 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Sep 24 03:41:32 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=de0b60ff
Add _sync_decorator module
Add functions that decorate coroutine methods and functions for
synchronous usage, allowing coroutines to smoothly blend with
synchronous code. This eliminates clutter that might otherwise
discourage the proliferation of coroutine usage for I/O bound tasks.
In the next commit, _sync_decorator will be used for smooth
integration of new classes that have coroutine methods.
Bug: https://bugs.gentoo.org/662070
Reviewed-by: Brian Dolbec gentoo.org>
Signed-off-by: Zac Medico gentoo.org>
.../tests/util/futures/test_compat_coroutine.py| 14 ++
lib/portage/util/futures/_sync_decorator.py| 54 ++
2 files changed, 68 insertions(+)
diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py
b/lib/portage/tests/util/futures/test_compat_coroutine.py
index b6f75b1a2..f96aa9be5 100644
--- a/lib/portage/tests/util/futures/test_compat_coroutine.py
+++ b/lib/portage/tests/util/futures/test_compat_coroutine.py
@@ -6,6 +6,7 @@ from portage.util.futures.compat_coroutine import (
coroutine,
coroutine_return,
)
+from portage.util.futures._sync_decorator import _sync_decorator, _sync_methods
from portage.tests import TestCase
@@ -161,3 +162,16 @@ class CompatCoroutineTestCase(TestCase):
loop.run_until_complete(asyncio.wait([writer, reader]))
self.assertEqual(reader.result(), values)
+
+ # Test decoration of coroutine methods and functions for
+ # synchronous usage, allowing coroutines to smoothly
+ # blend with synchronous code.
+ sync_cubby = _sync_methods(cubby, loop=loop)
+ sync_reader = _sync_decorator(reader_coroutine, loop=loop)
+ writer = asyncio.ensure_future(writer_coroutine(cubby, values,
None), loop=loop)
+ self.assertEqual(sync_reader(cubby, None), values)
+ self.assertTrue(writer.done())
+
+ for i in range(3):
+ sync_cubby.write(i)
+ self.assertEqual(sync_cubby.read(), i)
diff --git a/lib/portage/util/futures/_sync_decorator.py
b/lib/portage/util/futures/_sync_decorator.py
new file mode 100644
index 0..02a0963a7
--- /dev/null
+++ b/lib/portage/util/futures/_sync_decorator.py
@@ -0,0 +1,54 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import functools
+
+import portage
+portage.proxy.lazyimport.lazyimport(globals(),
+ 'portage.util.futures:asyncio',
+)
+
+
+def _sync_decorator(func, loop=None):
+ """
+ Decorate an asynchronous function (either a corouting function or a
+ function that returns a Future) with a wrapper that runs the function
+ synchronously.
+ """
+ loop = asyncio._wrap_loop(loop)
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ return loop.run_until_complete(func(*args, **kwargs))
+ return wrapper
+
+
+def _sync_methods(obj, loop=None):
+ """
+ For use with synchronous code that needs to interact with an object
+ that has coroutine methods, this function generates a proxy which
+ conveniently converts coroutine methods into synchronous methods.
+ This allows coroutines to smoothly blend with synchronous
+ code, eliminating clutter that might otherwise discourage the
+ proliferation of coroutine usage for I/O bound tasks.
+ """
+ loop = asyncio._wrap_loop(loop)
+ return _ObjectAttrWrapper(obj,
+ lambda attr: _sync_decorator(attr, loop=loop)
+ if asyncio.iscoroutinefunction(attr) else attr)
+
+
+class _ObjectAttrWrapper(portage.proxy.objectproxy.ObjectProxy):
+
+ __slots__ = ('_obj', '_attr_wrapper')
+
+ def __init__(self, obj, attr_wrapper):
+ object.__setattr__(self, '_obj', obj)
+ object.__setattr__(self, '_attr_wrapper', attr_wrapper)
+
+ def __getattribute__(self, attr):
+ obj = object.__getattribute__(self, '_obj')
+ attr_wrapper = object.__getattribute__(self, '_attr_wrapper')
+ return attr_wrapper(getattr(obj, attr))
+
+ def _get_target(self):
+ return object.__getattribute__(self, '_obj')
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/
commit: b37256a524a0fbf88ffad20c9f01aaf37409ec66 Author: Zac Medico gentoo org> AuthorDate: Mon Sep 3 20:16:37 2018 + Commit: Zac Medico gentoo org> CommitDate: Mon Sep 3 20:20:40 2018 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b37256a5 compat_coroutine: CancelledError cancels coroutine's future lib/portage/tests/util/futures/test_compat_coroutine.py | 8 ++-- lib/portage/util/futures/compat_coroutine.py| 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py b/lib/portage/tests/util/futures/test_compat_coroutine.py index cbc070869..b6f75b1a2 100644 --- a/lib/portage/tests/util/futures/test_compat_coroutine.py +++ b/lib/portage/tests/util/futures/test_compat_coroutine.py @@ -71,6 +71,10 @@ class CompatCoroutineTestCase(TestCase): loop.run_until_complete, future) def test_cancelled_future(self): + """ + When a coroutine raises CancelledError, the coroutine's + future is cancelled. + """ @coroutine def cancelled_future_coroutine(loop=None): @@ -81,8 +85,8 @@ class CompatCoroutineTestCase(TestCase): yield future loop = asyncio.get_event_loop() - self.assertRaises(asyncio.CancelledError, - loop.run_until_complete, cancelled_future_coroutine(loop=loop)) + future = loop.run_until_complete(asyncio.wait([cancelled_future_coroutine()]))[0].pop() + self.assertTrue(future.cancelled()) def test_yield_expression_result(self): @coroutine diff --git a/lib/portage/util/futures/compat_coroutine.py b/lib/portage/util/futures/compat_coroutine.py index 59fdc31b6..3edfa6bee 100644 --- a/lib/portage/util/futures/compat_coroutine.py +++ b/lib/portage/util/futures/compat_coroutine.py @@ -102,6 +102,8 @@ class _GeneratorTask(object): self._generator.throw(previous.exception()) future = next(self._generator) + except asyncio.CancelledError: + self._result.cancel() except _CoroutineReturnValue as e: if not self._result.cancelled(): self._result.set_result(e.result)
