commit: c95fc64abf9698263090b3ffd4a056e989dd2be1 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Fri Feb 9 06:38:41 2024 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Fri Feb 9 08:19:00 2024 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=c95fc64a
EbuildPhase: async_check_locale Change config.environ() check_locale calls to async_check_locale calls in the EbuildPhase _async_start method in order to eliminate synchronous waiting for child processes in the main event loop thread. Bug: https://bugs.gentoo.org/923841 Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> lib/_emerge/EbuildMetadataPhase.py | 21 ++++++++++++++++++++ lib/_emerge/EbuildPhase.py | 28 ++++++++++++++++++++++++++- lib/portage/package/ebuild/config.py | 26 +++++++++++-------------- lib/portage/util/futures/_asyncio/__init__.py | 9 +++++++++ lib/portage/util/locale.py | 28 ++++++++++++++++++--------- 5 files changed, 87 insertions(+), 25 deletions(-) diff --git a/lib/_emerge/EbuildMetadataPhase.py b/lib/_emerge/EbuildMetadataPhase.py index f4f685e81c..53b7ad9624 100644 --- a/lib/_emerge/EbuildMetadataPhase.py +++ b/lib/_emerge/EbuildMetadataPhase.py @@ -8,12 +8,14 @@ import portage portage.proxy.lazyimport.lazyimport( globals(), + "_emerge.EbuildPhase:_setup_locale", "portage.package.ebuild._metadata_invalid:eapi_invalid", ) from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode +from portage.util.futures import asyncio import fcntl @@ -44,6 +46,12 @@ class EbuildMetadataPhase(SubProcess): _files_dict = slot_dict_class(_file_names, prefix="") def _start(self): + asyncio.ensure_future( + self._async_start(), loop=self.scheduler + ).add_done_callback(self._async_start_done) + + async def _async_start(self): + ebuild_path = self.ebuild_hash.location with open( @@ -75,6 +83,9 @@ class EbuildMetadataPhase(SubProcess): settings.setcpv(self.cpv) settings.configdict["pkg"]["EAPI"] = parsed_eapi + # This requires above setcpv and EAPI setup. + await _setup_locale(self.settings) + debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None slave_fd = None @@ -139,6 +150,16 @@ class EbuildMetadataPhase(SubProcess): self._proc = retval + def _async_start_done(self, future): + future.cancelled() or future.result() + if future.cancelled(): + self.cancel() + self._was_cancelled() + + if self.returncode is not None: + self._unregister() + self.wait() + def _output_handler(self): while True: buf = self._read_buf(self._files.ebuild) diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py index c81bf54a81..c8caf73722 100644 --- a/lib/_emerge/EbuildPhase.py +++ b/lib/_emerge/EbuildPhase.py @@ -1,4 +1,4 @@ -# Copyright 1999-2021 Gentoo Authors +# Copyright 1999-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import functools @@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import ( _prepare_fake_distdir, _prepare_fake_filesdir, ) +from portage.eapi import _get_eapi_attrs from portage.util import writemsg, ensure_dirs from portage.util._async.AsyncTaskFuture import AsyncTaskFuture from portage.util._async.BuildLogger import BuildLogger @@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport( + "_post_src_install_write_metadata," + "_preinst_bsdflags", "portage.util.futures.unix_events:_set_nonblocking", + "portage.util.locale:async_check_locale,split_LC_ALL", ) from portage import os from portage import _encodings from portage import _unicode_encode +async def _setup_locale(settings): + eapi_attrs = _get_eapi_attrs(settings["EAPI"]) + if eapi_attrs.posixish_locale: + split_LC_ALL(settings) + settings["LC_COLLATE"] = "C" + # check_locale() returns None when check can not be executed. + if await async_check_locale(silent=True, env=settings.environ()) is False: + # try another locale + for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"): + settings["LC_CTYPE"] = l + if await async_check_locale(silent=True, env=settings.environ()): + # TODO: output the following only once + # writemsg( + # _("!!! LC_CTYPE unsupported, using %s instead\n") + # % self.settings["LC_CTYPE"] + # ) + break + else: + raise AssertionError("C locale did not pass the test!") + + class EbuildPhase(CompositeTask): __slots__ = ("actionmap", "fd_pipes", "phase", "settings") + ("_ebuild_lock",) @@ -94,6 +117,9 @@ class EbuildPhase(CompositeTask): self._start_task(AsyncTaskFuture(future=future), self._async_start_exit) async def _async_start(self): + + await _setup_locale(self.settings) + need_builddir = self.phase not in EbuildProcess._phases_without_builddir if need_builddir: diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py index d7b0ca5676..35c77486ec 100644 --- a/lib/portage/package/ebuild/config.py +++ b/lib/portage/package/ebuild/config.py @@ -29,7 +29,6 @@ portage.proxy.lazyimport.lazyimport( "portage.dbapi.vartree:vartree", "portage.package.ebuild.doebuild:_phase_func_map", "portage.util.compression_probe:_compressors", - "portage.util.locale:check_locale,split_LC_ALL", ) from portage import bsd_chflags, load_mod, os, selinux, _unicode_decode from portage.const import ( @@ -3368,20 +3367,17 @@ class config: mydict["EBUILD_PHASE_FUNC"] = phase_func if eapi_attrs.posixish_locale: - split_LC_ALL(mydict) - mydict["LC_COLLATE"] = "C" - # check_locale() returns None when check can not be executed. - if check_locale(silent=True, env=mydict) is False: - # try another locale - for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"): - mydict["LC_CTYPE"] = l - if check_locale(silent=True, env=mydict): - # TODO: output the following only once - # writemsg(_("!!! LC_CTYPE unsupported, using %s instead\n") - # % mydict["LC_CTYPE"]) - break - else: - raise AssertionError("C locale did not pass the test!") + if mydict.get("LC_ALL"): + # Sometimes this method is called for processes + # that are not ebuild phases, so only raise + # AssertionError for actual ebuild phases. + if phase and phase not in ("clean", "cleanrm", "fetch"): + raise AssertionError( + f"LC_ALL={mydict['LC_ALL']} for posixish locale. It seems that split_LC_ALL was not called for phase {phase}?" + ) + elif "LC_ALL" in mydict: + # Delete placeholder from split_LC_ALL. + del mydict["LC_ALL"] if not eapi_attrs.exports_PORTDIR: mydict.pop("PORTDIR", None) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index 8f1b8e8275..e78686bc52 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -15,6 +15,7 @@ __all__ = ( "set_child_watcher", "get_event_loop_policy", "set_event_loop_policy", + "run", "shield", "sleep", "Task", @@ -106,6 +107,14 @@ def set_child_watcher(watcher): return get_event_loop_policy().set_child_watcher(watcher) +# Emulate run since it's the preferred python API. +def run(coro): + return _safe_loop().run_until_complete(coro) + + +run.__doc__ = _real_asyncio.run.__doc__ + + def create_subprocess_exec(*args, **kwargs): """ Create a subprocess. diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py index b5da8d949b..b6a41e7655 100644 --- a/lib/portage/util/locale.py +++ b/lib/portage/util/locale.py @@ -17,6 +17,7 @@ import traceback import portage from portage.util import _unicode_decode, writemsg_level from portage.util._ctypes import find_library, LoadLibrary +from portage.util.futures import asyncio locale_categories = ( @@ -121,7 +122,10 @@ def check_locale(silent=False, env=None): warning and returns False if it is not. Returns None if the check can not be executed due to platform limitations. """ + return asyncio.run(async_check_locale(silent=silent, env=env)) + +async def async_check_locale(silent=False, env=None): if env is not None: for v in ("LC_ALL", "LC_CTYPE", "LANG"): if v in env: @@ -135,20 +139,17 @@ def check_locale(silent=False, env=None): except KeyError: pass - # TODO: Make async version of check_locale and call it from - # EbuildPhase instead of config.environ(), since it's bad to - # synchronously wait for the process in the main event loop - # thread where config.environ() tends to be called. proc = multiprocessing.Process( target=_set_and_check_locale, args=(silent, env, None if env is None else portage._native_string(mylocale)), ) proc.start() - proc.join() + proc = portage.process.MultiprocessingProcess(proc) + await proc.wait() pyret = None - if proc.exitcode >= 0: - ret = proc.exitcode + if proc.returncode >= 0: + ret = proc.returncode if ret != 2: pyret = ret == 0 @@ -157,13 +158,22 @@ def check_locale(silent=False, env=None): return pyret +async_check_locale.__doc__ = check_locale.__doc__ +async_check_locale.__doc__ += """ + This function is a coroutine. +""" + + def split_LC_ALL(env): """ Replace LC_ALL with split-up LC_* variables if it is defined. Works on the passed environment (or settings instance). """ lc_all = env.get("LC_ALL") - if lc_all is not None: + if lc_all: for c in locale_categories: env[c] = lc_all - del env["LC_ALL"] + # Set empty so that config.reset() can restore LC_ALL state, + # since del can permanently delete variables which are not + # stored in the config's backupenv. + env["LC_ALL"] = ""
