commit: 2d500ce2bc96995752dfc2fb475a7abe907e38b6 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Sun May 6 21:28:25 2018 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Sun May 6 22:51:23 2018 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=2d500ce2
asyncio: explicitly close event loops (bug 654390) The default asyncio event loop triggers a resource warning if it is not explicitly closed, therefore close it when appropriate. Bug: https://bugs.gentoo.org/654390 bin/ebuild | 4 ++++ bin/ebuild-ipc.py | 5 ++++- bin/egencache | 5 ++++- bin/emaint | 3 +++ bin/emerge | 5 +++++ bin/emirrordist | 6 +++++- bin/portageq | 6 +++++- bin/quickpkg | 3 +++ pym/portage/tests/runTests.py | 6 +++++- pym/portage/tests/util/futures/asyncio/test_child_watcher.py | 5 +++++ pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py | 6 ++++++ pym/portage/tests/util/futures/asyncio/test_pipe_closed.py | 7 +++++++ pym/portage/tests/util/futures/asyncio/test_run_until_complete.py | 5 +++++ pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py | 7 ++++++- pym/portage/tests/util/futures/test_retry.py | 5 ++++- pym/portage/util/_eventloop/global_event_loop.py | 5 ++++- 16 files changed, 75 insertions(+), 8 deletions(-) diff --git a/bin/ebuild b/bin/ebuild index 5221b21a8..710257549 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -55,6 +55,7 @@ from portage.exception import PermissionDenied, PortageKeyError, \ PortagePackageException, UnsupportedAPIException from portage.localization import _ import portage.util +from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.Package import Package from _emerge.RootConfig import RootConfig @@ -371,4 +372,7 @@ for arg in pargs: print("Could not run the required binary?") a = 127 if a: + global_event_loop().close() sys.exit(a) + +global_event_loop().close() diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py index 6d0cdbef9..1f323bdc5 100755 --- a/bin/ebuild-ipc.py +++ b/bin/ebuild-ipc.py @@ -273,4 +273,7 @@ def ebuild_ipc_main(args): return ebuild_ipc.communicate(args) if __name__ == '__main__': - sys.exit(ebuild_ipc_main(sys.argv[1:])) + try: + sys.exit(ebuild_ipc_main(sys.argv[1:])) + finally: + global_event_loop().close() diff --git a/bin/egencache b/bin/egencache index e994b4ab1..70fb5aa00 100755 --- a/bin/egencache +++ b/bin/egencache @@ -1116,4 +1116,7 @@ def egencache_main(args): if __name__ == "__main__": portage._disable_legacy_globals() portage.util.noiselimit = -1 - sys.exit(egencache_main(sys.argv[1:])) + try: + sys.exit(egencache_main(sys.argv[1:])) + finally: + global_event_loop().close() diff --git a/bin/emaint b/bin/emaint index 08e75851a..a26dae1e7 100755 --- a/bin/emaint +++ b/bin/emaint @@ -31,6 +31,7 @@ if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".porta import portage portage._internal_caller = True from portage.emaint.main import emaint_main +from portage.util._eventloop.global_event_loop import global_event_loop try: emaint_main(sys.argv[1:]) @@ -40,3 +41,5 @@ except IOError as e: sys.exit(1) else: raise +finally: + global_event_loop().close() diff --git a/bin/emerge b/bin/emerge index 5f08861e5..e1262d544 100755 --- a/bin/emerge +++ b/bin/emerge @@ -13,6 +13,7 @@ import sys # exiting from signal handlers intermittently causes python to ignore # the SystemExit exception with a message like this: # Exception SystemExit: 130 in <function remove at 0x7fd2146c1320> ignored +global_event_loop = None try: def exithandler(signum, _frame): @@ -41,6 +42,7 @@ try: import portage portage._internal_caller = True portage._disable_legacy_globals() + from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.main import emerge_main if __name__ == "__main__": @@ -84,3 +86,6 @@ except KeyboardInterrupt: {"signal": signal.SIGINT}) sys.stderr.flush() sys.exit(128 + signal.SIGINT) +finally: + if global_event_loop is not None: + global_event_loop().close() diff --git a/bin/emirrordist b/bin/emirrordist index 17f99f590..3ea08379e 100755 --- a/bin/emirrordist +++ b/bin/emirrordist @@ -9,6 +9,7 @@ import portage portage._internal_caller = True portage._disable_legacy_globals() from portage._emirrordist.main import emirrordist_main +from portage.util._eventloop.global_event_loop import global_event_loop if __name__ == "__main__": @@ -18,4 +19,7 @@ if __name__ == "__main__": signal.signal(signal.SIGUSR1, debug_signal) - sys.exit(emirrordist_main(sys.argv[1:])) + try: + sys.exit(emirrordist_main(sys.argv[1:])) + finally: + global_event_loop().close() diff --git a/bin/portageq b/bin/portageq index e9b8b20e0..35499afd2 100755 --- a/bin/portageq +++ b/bin/portageq @@ -53,6 +53,7 @@ portage.proxy.lazyimport.lazyimport(globals(), '_emerge.is_valid_package_atom:insert_category_into_atom', 'portage.dbapi._expand_new_virt:expand_new_virt', 'portage._sets.base:InternalPackageSet', + 'portage.util._eventloop.global_event_loop:global_event_loop', 'portage.xml.metadata:MetaDataXML' ) @@ -1466,6 +1467,9 @@ def main(argv): sys.exit(1) if __name__ == '__main__': - sys.exit(main(sys.argv)) + try: + sys.exit(main(sys.argv)) + finally: + global_event_loop().close() #----------------------------------------------------------------------------- diff --git a/bin/quickpkg b/bin/quickpkg index 9765ec717..f071dd904 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -30,6 +30,8 @@ from portage.checksum import perform_md5 from portage._sets import load_default_config, SETPREFIX from portage.process import find_binary from portage.util.compression_probe import _compressors +from portage.util._eventloop.global_event_loop import global_event_loop + def quickpkg_atom(options, infos, arg, eout): settings = portage.settings @@ -390,4 +392,5 @@ if __name__ == "__main__": finally: os.umask(old_umask) signal.signal(signal.SIGWINCH, signal.SIG_DFL) + global_event_loop().close() sys.exit(retval) diff --git a/pym/portage/tests/runTests.py b/pym/portage/tests/runTests.py index 9c452764f..d4d1f7c76 100755 --- a/pym/portage/tests/runTests.py +++ b/pym/portage/tests/runTests.py @@ -42,6 +42,7 @@ if os.environ.get('NOCOLOR') in ('yes', 'true'): portage.output.nocolor() import portage.tests as tests +from portage.util._eventloop.global_event_loop import global_event_loop from portage.const import PORTAGE_BIN_PATH path = os.environ.get("PATH", "").split(":") path = [x for x in path if x] @@ -58,4 +59,7 @@ if insert_bin_path: os.environ["PATH"] = ":".join(path) if __name__ == "__main__": - sys.exit(tests.main()) + try: + sys.exit(tests.main()) + finally: + global_event_loop().close() diff --git a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py index 8ef497544..0fc73ab49 100644 --- a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py +++ b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py @@ -5,6 +5,7 @@ import os from portage.process import find_binary, spawn from portage.tests import TestCase +from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.futures import asyncio from portage.util.futures.unix_events import DefaultEventLoopPolicy @@ -18,6 +19,7 @@ class ChildWatcherTestCase(TestCase): if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) + loop = None try: try: asyncio.set_child_watcher(None) @@ -43,3 +45,6 @@ class ChildWatcherTestCase(TestCase): (pids[0], os.EX_OK, args_tuple)) finally: asyncio.set_event_loop_policy(initial_policy) + if loop not in (None, global_event_loop()): + loop.close() + self.assertFalse(global_event_loop().is_closed()) diff --git a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py index 19588bf3a..177953437 100644 --- a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py +++ b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py @@ -5,6 +5,7 @@ import multiprocessing import os from portage.tests import TestCase +from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.futures import asyncio from portage.util.futures.unix_events import DefaultEventLoopPolicy @@ -15,6 +16,7 @@ def fork_main(parent_conn, child_conn): # This fails with python's default event loop policy, # see https://bugs.python.org/issue22087. loop.run_until_complete(asyncio.sleep(0.1, loop=loop)) + loop.close() def async_main(fork_exitcode, loop=None): @@ -47,6 +49,7 @@ class EventLoopInForkTestCase(TestCase): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) + loop = None try: loop = asyncio._wrap_loop() fork_exitcode = loop.create_future() @@ -57,3 +60,6 @@ class EventLoopInForkTestCase(TestCase): assert loop.run_until_complete(fork_exitcode) == os.EX_OK finally: asyncio.set_event_loop_policy(initial_policy) + if loop not in (None, global_event_loop()): + loop.close() + self.assertFalse(global_event_loop().is_closed()) diff --git a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py index c2b468064..507385c04 100644 --- a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py +++ b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py @@ -10,6 +10,7 @@ import sys import tempfile from portage.tests import TestCase +from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.futures import asyncio from portage.util.futures.unix_events import ( DefaultEventLoopPolicy, @@ -83,6 +84,9 @@ class ReaderPipeClosedTestCase(_PipeClosedTestCase, TestCase): write_end.close() read_end.close() asyncio.set_event_loop_policy(initial_policy) + if loop not in (None, global_event_loop()): + loop.close() + self.assertFalse(global_event_loop().is_closed()) class WriterPipeClosedTestCase(_PipeClosedTestCase, TestCase): @@ -142,3 +146,6 @@ class WriterPipeClosedTestCase(_PipeClosedTestCase, TestCase): write_end.close() read_end.close() asyncio.set_event_loop_policy(initial_policy) + if loop not in (None, global_event_loop()): + loop.close() + self.assertFalse(global_event_loop().is_closed()) diff --git a/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py b/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py index 1a37e4922..c0e86ae5e 100644 --- a/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py +++ b/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase +from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.futures import asyncio from portage.util.futures.unix_events import DefaultEventLoopPolicy @@ -12,6 +13,7 @@ class RunUntilCompleteTestCase(TestCase): if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) + loop = None try: loop = asyncio._wrap_loop() f1 = loop.create_future() @@ -27,3 +29,6 @@ class RunUntilCompleteTestCase(TestCase): self.assertEqual(f2.done(), True) finally: asyncio.set_event_loop_policy(initial_policy) + if loop not in (None, global_event_loop()): + loop.close() + self.assertFalse(global_event_loop().is_closed()) diff --git a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py index 8dc5fa7b9..534d79c53 100644 --- a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py +++ b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py @@ -6,6 +6,7 @@ import subprocess from portage.process import find_binary from portage.tests import TestCase +from portage.util._eventloop.global_event_loop import global_event_loop from portage.util.futures import asyncio from portage.util.futures.executor.fork import ForkExecutor from portage.util.futures.unix_events import DefaultEventLoopPolicy @@ -60,10 +61,14 @@ class SubprocessExecTestCase(TestCase): if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) + loop = asyncio._wrap_loop() try: - test(asyncio._wrap_loop()) + test(loop) finally: asyncio.set_event_loop_policy(initial_policy) + if loop not in (None, global_event_loop()): + loop.close() + self.assertFalse(global_event_loop().is_closed()) def testEcho(self): if not hasattr(asyncio, 'create_subprocess_exec'): diff --git a/pym/portage/tests/util/futures/test_retry.py b/pym/portage/tests/util/futures/test_retry.py index 781eac9a1..baf293d56 100644 --- a/pym/portage/tests/util/futures/test_retry.py +++ b/pym/portage/tests/util/futures/test_retry.py @@ -183,7 +183,10 @@ class RetryExecutorTestCase(RetryTestCase): return result.result() else: # child process - return loop.run_until_complete(coroutine_func()) + try: + return loop.run_until_complete(coroutine_func()) + finally: + loop.close() def execute_wrapper(): kill_switch = parent_loop.create_future() diff --git a/pym/portage/util/_eventloop/global_event_loop.py b/pym/portage/util/_eventloop/global_event_loop.py index e2c7d71ea..a3ee9248d 100644 --- a/pym/portage/util/_eventloop/global_event_loop.py +++ b/pym/portage/util/_eventloop/global_event_loop.py @@ -29,6 +29,9 @@ def global_event_loop(): if not constructor.supports_multiprocessing and pid != _MAIN_PID: constructor = _multiprocessing_constructor - instance = constructor() + # Use the _asyncio_wrapper attribute, so that unit tests can compare + # the reference to one retured from _wrap_loop(), since they should + # not close the loop if it refers to a global event loop. + instance = constructor()._asyncio_wrapper _instances[pid] = instance return instance
