commit: 285d5d038d8bb8a17d853816e156147c8c59f248 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Mon Apr 3 00:40:55 2017 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Tue Apr 4 03:19:29 2017 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=285d5d03
EbuildBuild: async spawn_nofetch in _fetchonly_exit (bug 614116) Replace a synchronous spawn_nofetch call with an asynchronous one, in order to avoid event loop recursion which is not compatible with asyncio. This involves refactoring of spawn_nofetch to provide an asynchronous interface. X-Gentoo-bug: 614116 X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=614116 Acked-by: Brian Dolbec <dolsen <AT> gentoo.org> pym/_emerge/EbuildBuild.py | 18 ++++- pym/portage/package/ebuild/_spawn_nofetch.py | 106 +++++++++++++++++---------- 2 files changed, 85 insertions(+), 39 deletions(-) diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index 11eb1c93e..48f470483 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -22,7 +22,7 @@ import portage from portage import _encodings, _unicode_decode, _unicode_encode, os from portage.package.ebuild.digestcheck import digestcheck from portage.package.ebuild.doebuild import _check_temp_dir -from portage.package.ebuild._spawn_nofetch import spawn_nofetch +from portage.package.ebuild._spawn_nofetch import SpawnNofetchWithoutBuilddir class EbuildBuild(CompositeTask): @@ -165,8 +165,22 @@ class EbuildBuild(CompositeTask): def _fetchonly_exit(self, fetcher): self._final_exit(fetcher) if self.returncode != os.EX_OK: + self.returncode = None portdb = self.pkg.root_config.trees[self._tree].dbapi - spawn_nofetch(portdb, self._ebuild_path, settings=self.settings) + self._start_task(SpawnNofetchWithoutBuilddir( + background=self.background, + portdb=portdb, + ebuild_path=self._ebuild_path, + scheduler=self.scheduler, + settings=self.settings), + self._nofetch_without_builddir_exit) + return + + self.wait() + + def _nofetch_without_builddir_exit(self, nofetch): + self._final_exit(nofetch) + self.returncode = 1 self.wait() def _pre_clean_exit(self, pre_clean_phase): diff --git a/pym/portage/package/ebuild/_spawn_nofetch.py b/pym/portage/package/ebuild/_spawn_nofetch.py index 0fc53c8ca..bbfd5b72b 100644 --- a/pym/portage/package/ebuild/_spawn_nofetch.py +++ b/pym/portage/package/ebuild/_spawn_nofetch.py @@ -14,11 +14,14 @@ from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs from portage.util._async.SchedulerInterface import SchedulerInterface from portage.util._eventloop.EventLoop import EventLoop from portage.util._eventloop.global_event_loop import global_event_loop +from _emerge.CompositeTask import CompositeTask from _emerge.EbuildPhase import EbuildPhase -def spawn_nofetch(portdb, ebuild_path, settings=None, fd_pipes=None): + +class SpawnNofetchWithoutBuilddir(CompositeTask): """ - This spawns pkg_nofetch if appropriate. The settings parameter + This spawns pkg_nofetch if appropriate, while avoiding the + need to lock a global build directory. The settings parameter is useful only if setcpv has already been called in order to cache metadata. It will be cloned internally, in order to prevent any changes from interfering with the calling code. @@ -40,33 +43,42 @@ def spawn_nofetch(portdb, ebuild_path, settings=None, fd_pipes=None): to be displayed for problematic packages even though they do not set RESTRICT=fetch (bug #336499). - This function does nothing if the PORTAGE_PARALLEL_FETCHONLY + This class does nothing if the PORTAGE_PARALLEL_FETCHONLY variable is set in the config instance. """ + __slots__ = ('ebuild_path', 'fd_pipes', 'portdb', 'settings', + '_private_tmpdir') + + def _start(self): + settings = self.settings + if settings is None: + settings = self.portdb.settings + + if 'PORTAGE_PARALLEL_FETCHONLY' in settings: + # parallel-fetch mode + self.returncode = os.EX_OK + self._async_wait() + return - if settings is None: - settings = config(clone=portdb.settings) - else: - settings = config(clone=settings) - - if 'PORTAGE_PARALLEL_FETCHONLY' in settings: - return os.EX_OK - - # We must create our private PORTAGE_TMPDIR before calling - # doebuild_environment(), since lots of variables such - # as PORTAGE_BUILDDIR refer to paths inside PORTAGE_TMPDIR. - portage_tmpdir = settings.get('PORTAGE_TMPDIR') - if not portage_tmpdir or not os.access(portage_tmpdir, os.W_OK): - portage_tmpdir = None - private_tmpdir = tempfile.mkdtemp(dir=portage_tmpdir) - settings['PORTAGE_TMPDIR'] = private_tmpdir - settings.backup_changes('PORTAGE_TMPDIR') - # private temp dir was just created, so it's not locked yet - settings.pop('PORTAGE_BUILDDIR_LOCKED', None) - - try: - doebuild_environment(ebuild_path, 'nofetch', - settings=settings, db=portdb) + # Prevent temporary config changes from interfering + # with config instances that are reused. + settings = self.settings = config(clone=settings) + + # We must create our private PORTAGE_TMPDIR before calling + # doebuild_environment(), since lots of variables such + # as PORTAGE_BUILDDIR refer to paths inside PORTAGE_TMPDIR. + portage_tmpdir = settings.get('PORTAGE_TMPDIR') + if not portage_tmpdir or not os.access(portage_tmpdir, os.W_OK): + portage_tmpdir = None + private_tmpdir = self._private_tmpdir = tempfile.mkdtemp( + dir=portage_tmpdir) + settings['PORTAGE_TMPDIR'] = private_tmpdir + settings.backup_changes('PORTAGE_TMPDIR') + # private temp dir was just created, so it's not locked yet + settings.pop('PORTAGE_BUILDDIR_LOCKED', None) + + doebuild_environment(self.ebuild_path, 'nofetch', + settings=settings, db=self.portdb) restrict = settings['PORTAGE_RESTRICT'].split() defined_phases = settings['DEFINED_PHASES'].split() if not defined_phases: @@ -76,18 +88,38 @@ def spawn_nofetch(portdb, ebuild_path, settings=None, fd_pipes=None): if 'fetch' not in restrict and \ 'nofetch' not in defined_phases: - return os.EX_OK + self.returncode = os.EX_OK + self._async_wait() + return prepare_build_dirs(settings=settings) - ebuild_phase = EbuildPhase(background=False, + + ebuild_phase = EbuildPhase(background=self.background, phase='nofetch', - scheduler=SchedulerInterface(portage._internal_caller and + scheduler=self.scheduler, + fd_pipes=self.fd_pipes, settings=settings) + + self._start_task(ebuild_phase, self._nofetch_exit) + + def _nofetch_exit(self, ebuild_phase): + self._final_exit(ebuild_phase) + elog_process(self.settings.mycpv, self.settings) + shutil.rmtree(self._private_tmpdir) + self._async_wait() + + +def spawn_nofetch(portdb, ebuild_path, settings=None, fd_pipes=None): + """ + Create a NofetchPrivateTmpdir instance, and execute it synchronously. + This function must not be called from asynchronous code, since it will + trigger event loop recursion which is incompatible with asyncio. + """ + nofetch = SpawnNofetchWithoutBuilddir(background=False, + portdb=portdb, + ebuild_path=ebuild_path, + scheduler=SchedulerInterface(portage._internal_caller and global_event_loop() or EventLoop(main=False)), - fd_pipes=fd_pipes, settings=settings) - ebuild_phase.start() - ebuild_phase.wait() - elog_process(settings.mycpv, settings) - finally: - shutil.rmtree(private_tmpdir) - - return ebuild_phase.returncode + fd_pipes=fd_pipes, settings=settings) + + nofetch.start() + return nofetch.wait()
