Re: [gentoo-portage-dev] [PATCH 2/2] Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)
On 6/22/20 7:46 AM, Brian Dolbec wrote: > > That's a lot of code...but I couldn't spot anything wrong, so looks good Thanks, merged: https://gitweb.gentoo.org/proj/portage.git/commit/?id=dd69ce742c62b9515cf7ae37e46bcf7f178777db -- Thanks, Zac signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH 2/2] Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)
On Fri, 19 Jun 2020 13:39:19 -0700
Zac Medico wrote:
> This variable specifies a command that filters build log output to a
> log file. The plan is to extend this to support a separate filter for
> tty output in the future.
>
> In order to enable the EbuildPhase class to write elog messages to
> the build log with PORTAGE_LOG_FILTER_FILE_CMD support, convert its
> _elog method to a coroutine, and add a SchedulerInterface async_output
> method for it to use.
>
> Use a new BuildLogger class to manage log output (with or without a
> filter command), with compression support provided by PipeLogger.
> BuildLogger has a stdin property which provides access to a writable
> binary file stream (refers to a pipe) that log content is written to.
>
> Bug: https://bugs.gentoo.org/709746
> Signed-off-by: Zac Medico
> ---
> lib/_emerge/AbstractEbuildProcess.py | 3 +-
> lib/_emerge/BinpkgFetcher.py | 3 +-
> lib/_emerge/EbuildFetcher.py | 3 +-
> lib/_emerge/EbuildPhase.py| 47 ++--
> lib/_emerge/SpawnProcess.py | 58 +++---
> lib/portage/dbapi/_MergeProcess.py| 3 +-
> .../ebuild/_config/special_env_vars.py| 8 +-
> lib/portage/util/_async/BuildLogger.py| 109
> ++ lib/portage/util/_async/SchedulerInterface.py |
> 32 - man/make.conf.5 | 7 +-
> 10 files changed, 243 insertions(+), 30 deletions(-)
> create mode 100644 lib/portage/util/_async/BuildLogger.py
>
> diff --git a/lib/_emerge/AbstractEbuildProcess.py
> b/lib/_emerge/AbstractEbuildProcess.py index 1c1955cfe..ae1aae55f
> 100644 --- a/lib/_emerge/AbstractEbuildProcess.py
> +++ b/lib/_emerge/AbstractEbuildProcess.py
> @@ -1,4 +1,4 @@
> -# Copyright 1999-2019 Gentoo Foundation
> +# Copyright 1999-2020 Gentoo Authors
> # Distributed under the terms of the GNU General Public License v2
>
> import errno
> @@ -196,6 +196,7 @@ class AbstractEbuildProcess(SpawnProcess):
> null_fd = os.open('/dev/null', os.O_RDONLY)
> self.fd_pipes[0] = null_fd
>
> + self.log_filter_file =
> self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD') try:
> SpawnProcess._start(self)
> finally:
> diff --git a/lib/_emerge/BinpkgFetcher.py
> b/lib/_emerge/BinpkgFetcher.py index 36d027de3..2e5861cc1 100644
> --- a/lib/_emerge/BinpkgFetcher.py
> +++ b/lib/_emerge/BinpkgFetcher.py
> @@ -1,4 +1,4 @@
> -# Copyright 1999-2018 Gentoo Foundation
> +# Copyright 1999-2020 Gentoo Authors
> # Distributed under the terms of the GNU General Public License v2
>
> import functools
> @@ -158,6 +158,7 @@ class _BinpkgFetcherProcess(SpawnProcess):
> self.env = fetch_env
> if settings.selinux_enabled():
> self._selinux_type =
> settings["PORTAGE_FETCH_T"]
> + self.log_filter_file =
> settings.get('PORTAGE_LOG_FILTER_FILE_CMD') SpawnProcess._start(self)
>
> def _pipe(self, fd_pipes):
> diff --git a/lib/_emerge/EbuildFetcher.py
> b/lib/_emerge/EbuildFetcher.py index 1e40994fb..55349c33c 100644
> --- a/lib/_emerge/EbuildFetcher.py
> +++ b/lib/_emerge/EbuildFetcher.py
> @@ -1,4 +1,4 @@
> -# Copyright 1999-2018 Gentoo Foundation
> +# Copyright 1999-2020 Gentoo Authors
> # Distributed under the terms of the GNU General Public License v2
>
> import copy
> @@ -225,6 +225,7 @@ class _EbuildFetcherProcess(ForkProcess):
> settings["NOCOLOR"] = nocolor
>
> self._settings = settings
> + self.log_filter_file =
> settings.get('PORTAGE_LOG_FILTER_FILE_CMD') ForkProcess._start(self)
>
> # Free settings now since it's no longer needed in
> diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
> index 477e0ba97..ddb3dc719 100644
> --- a/lib/_emerge/EbuildPhase.py
> +++ b/lib/_emerge/EbuildPhase.py
> @@ -26,6 +26,8 @@ from portage.package.ebuild.prepare_build_dirs
> import (_prepare_workdir, from portage.util.futures.compat_coroutine
> import coroutine from portage.util import writemsg
> from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
> +from portage.util._async.BuildLogger import BuildLogger
> +from portage.util.futures import asyncio
> from portage.util.futures.executor.fork import ForkExecutor
>
> try:
> @@ -69,6 +71,11 @@ class EbuildPhase(CompositeTask):
> _locked_phases = ("setup", "preinst", "postinst", "prerm",
> "postrm")
> def _start(self):
> + future = asyncio.ensure_future(self._async_start(),
> loop=self.scheduler)
> + self._start_task(AsyncTaskFuture(future=future),
> self._async_start_exit) +
> + @coroutine
> + def _async_start(self):
>
> need_builddir = self.phase not in
> EbuildProcess._phases_without_builddir
> @@ -126,7 +133,7 @@ class EbuildPhase(CompositeTask):
> # Force
Re: [gentoo-portage-dev] [PATCH 1/2] PipeLogger: non-blocking write to pipe (bug 709746)
On Fri, 19 Jun 2020 13:39:18 -0700
Zac Medico wrote:
> Add support to write to a non-blocking pipe instead of a
> log file. This is needed for the purposes of bug 709746,
> where PipeLogger will write to a pipe that is drained
> by anoher PipeLogger instance which is running in the same
> process.
>
> Bug: https://bugs.gentoo.org/709746
> Signed-off-by: Zac Medico
> ---
> lib/portage/tests/process/test_PipeLogger.py | 58
> lib/portage/util/_async/PipeLogger.py| 73
> +++- 2 files changed, 115 insertions(+), 16
> deletions(-) create mode 100644
> lib/portage/tests/process/test_PipeLogger.py
>
> diff --git a/lib/portage/tests/process/test_PipeLogger.py
> b/lib/portage/tests/process/test_PipeLogger.py new file mode 100644
> index 0..2bd94cf39
> --- /dev/null
> +++ b/lib/portage/tests/process/test_PipeLogger.py
> @@ -0,0 +1,58 @@
> +# Copyright 2020 Gentoo Authors
> +# Distributed under the terms of the GNU General Public License v2
> +
> +from portage import os
> +from portage.tests import TestCase
> +from portage.util._async.PipeLogger import PipeLogger
> +from portage.util.futures import asyncio
> +from portage.util.futures._asyncio.streams import _reader, _writer
> +from portage.util.futures.compat_coroutine import coroutine,
> coroutine_return +from portage.util.futures.unix_events import
> _set_nonblocking +
> +
> +class PipeLoggerTestCase(TestCase):
> +
> + @coroutine
> + def _testPipeLoggerToPipe(self, test_string, loop=None):
> + """
> + Test PipeLogger writing to a pipe connected to a
> PipeReader.
> + This verifies that PipeLogger does not deadlock when
> writing
> + to a pipe that's drained by a PipeReader running in
> the same
> + process (requires non-blocking write).
> + """
> +
> + input_fd, writer_pipe = os.pipe()
> + _set_nonblocking(writer_pipe)
> + writer_pipe = os.fdopen(writer_pipe, 'wb', 0)
> + writer = asyncio.ensure_future(_writer(writer_pipe,
> test_string.encode('ascii'), loop=loop), loop=loop)
> + writer.add_done_callback(lambda writer:
> writer_pipe.close()) +
> + pr, pw = os.pipe()
> +
> + consumer = PipeLogger(background=True,
> + input_fd=input_fd,
> + log_file_path=os.fdopen(pw, 'wb', 0),
> + scheduler=loop)
> + consumer.start()
> +
> + # Before starting the reader, wait here for a
> moment, in order
> + # to exercise PipeLogger's handling of EAGAIN during
> write.
> + yield asyncio.wait([writer], timeout=0.01)
> +
> + reader = _reader(pr, loop=loop)
> + yield writer
> + content = yield reader
> + yield consumer.async_wait()
> +
> + self.assertEqual(consumer.returncode, os.EX_OK)
> +
> + coroutine_return(content.decode('ascii', 'replace'))
> +
> + def testPipeLogger(self):
> + loop = asyncio._wrap_loop()
> +
> + for x in (1, 2, 5, 6, 7, 8, 2**5, 2**10, 2**12,
> 2**13, 2**14, 2**17, 2**17 + 1):
> + test_string = x * "a"
> + output =
> loop.run_until_complete(self._testPipeLoggerToPipe(test_string,
> loop=loop))
> + self.assertEqual(test_string, output,
> + "x = %s, len(output) = %s" % (x,
> len(output))) diff --git a/lib/portage/util/_async/PipeLogger.py
> b/lib/portage/util/_async/PipeLogger.py index a4258f350..ce8afb846
> 100644 --- a/lib/portage/util/_async/PipeLogger.py
> +++ b/lib/portage/util/_async/PipeLogger.py
> @@ -8,6 +8,10 @@ import sys
>
> import portage
> from portage import os, _encodings, _unicode_encode
> +from portage.util.futures import asyncio
> +from portage.util.futures._asyncio.streams import _writer
> +from portage.util.futures.compat_coroutine import coroutine
> +from portage.util.futures.unix_events import _set_nonblocking
> from _emerge.AbstractPollTask import AbstractPollTask
>
> class PipeLogger(AbstractPollTask):
> @@ -21,13 +25,16 @@ class PipeLogger(AbstractPollTask):
> """
>
> __slots__ = ("input_fd", "log_file_path", "stdout_fd") + \
> - ("_log_file", "_log_file_real")
> + ("_io_loop_task", "_log_file", "_log_file_nb",
> "_log_file_real")
> def _start(self):
>
> log_file_path = self.log_file_path
> - if log_file_path is not None:
> -
> + if hasattr(log_file_path, 'write'):
> + self._log_file_nb = True
> + self._log_file = log_file_path
> + _set_nonblocking(self._log_file.fileno())
> + elif log_file_path is not None:
> self._log_file =
> open(_unicode_encode(log_file_path, encoding=_encodings['fs'],
> errors='strict'), mode='ab') if
