[gentoo-dev] */*: Mask Py2 only packages

2020-06-19 Thread Aaron Bauman
> # Aaron Bauman  (2020-06-20)
> # Py2 only
> # Removal in 14 days
> app-admin/clustershell
> app-crypt/openvpn-blacklist
> app-forensics/volatility
> app-misc/golly
> app-misc/yagtd
> app-text/pylize
> app-text/rpl
> app-vim/easytags
> app-vim/notes
> dev-db/metakit
> dev-python/backports-ssl-match-hostname
> dev-python/backports-shutil_get_terminal_size
> dev-python/backports-shutil_which
> dev-python/python-wpactrl
> dev-util/bakefile
> dev-util/uftrace
> dev-vcs/git-deps
> dev-vcs/git-remote-hg
> games-board/scid
> games-emulation/openmsx
> games-kids/childsplay
> games-mud/lyntin
> games-sports/ski
> mail-filter/tmda
> media-video/subdl
> media-sound/cplay
> media-sound/tunapie
> net-firewall/dshieldpy
> net-mail/offlineimap
> net-misc/switzerland
> net-wireless/multimode
> sci-biology/last
> sci-chemistry/hollow
> sci-chemistry/modeller
> sci-chemistry/pymol-plugins-msms
> sci-geosciences/tilecache
> sci-libs/deap
> sci-libs/pycifrw
> sys-cluster/zookeeper-bin
> www-apps/curator
> x11-apps/whyteboard
> x11-plugins/purple-plugin_pack

-- 
Cheers,
Aaron


signature.asc
Description: PGP signature


[gentoo-portage-dev] [PATCH 1/2] PipeLogger: non-blocking write to pipe (bug 709746)

2020-06-19 Thread Zac Medico
Add support to write to a non-blocking pipe instead of a
log file. This is needed for the purposes of bug 709746,
where PipeLogger will write to a pipe that is drained
by anoher PipeLogger instance which is running in the same
process.

Bug: https://bugs.gentoo.org/709746
Signed-off-by: Zac Medico 
---
 lib/portage/tests/process/test_PipeLogger.py | 58 
 lib/portage/util/_async/PipeLogger.py| 73 +++-
 2 files changed, 115 insertions(+), 16 deletions(-)
 create mode 100644 lib/portage/tests/process/test_PipeLogger.py

diff --git a/lib/portage/tests/process/test_PipeLogger.py 
b/lib/portage/tests/process/test_PipeLogger.py
new file mode 100644
index 0..2bd94cf39
--- /dev/null
+++ b/lib/portage/tests/process/test_PipeLogger.py
@@ -0,0 +1,58 @@
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from portage import os
+from portage.tests import TestCase
+from portage.util._async.PipeLogger import PipeLogger
+from portage.util.futures import asyncio
+from portage.util.futures._asyncio.streams import _reader, _writer
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
+from portage.util.futures.unix_events import _set_nonblocking
+
+
+class PipeLoggerTestCase(TestCase):
+
+   @coroutine
+   def _testPipeLoggerToPipe(self, test_string, loop=None):
+   """
+   Test PipeLogger writing to a pipe connected to a PipeReader.
+   This verifies that PipeLogger does not deadlock when writing
+   to a pipe that's drained by a PipeReader running in the same
+   process (requires non-blocking write).
+   """
+
+   input_fd, writer_pipe = os.pipe()
+   _set_nonblocking(writer_pipe)
+   writer_pipe = os.fdopen(writer_pipe, 'wb', 0)
+   writer = asyncio.ensure_future(_writer(writer_pipe, 
test_string.encode('ascii'), loop=loop), loop=loop)
+   writer.add_done_callback(lambda writer: writer_pipe.close())
+
+   pr, pw = os.pipe()
+
+   consumer = PipeLogger(background=True,
+   input_fd=input_fd,
+   log_file_path=os.fdopen(pw, 'wb', 0),
+   scheduler=loop)
+   consumer.start()
+
+   # Before starting the reader, wait here for a moment, in order
+   # to exercise PipeLogger's handling of EAGAIN during write.
+   yield asyncio.wait([writer], timeout=0.01)
+
+   reader = _reader(pr, loop=loop)
+   yield writer
+   content = yield reader
+   yield consumer.async_wait()
+
+   self.assertEqual(consumer.returncode, os.EX_OK)
+
+   coroutine_return(content.decode('ascii', 'replace'))
+
+   def testPipeLogger(self):
+   loop = asyncio._wrap_loop()
+
+   for x in (1, 2, 5, 6, 7, 8, 2**5, 2**10, 2**12, 2**13, 2**14, 
2**17, 2**17 + 1):
+   test_string = x * "a"
+   output = 
loop.run_until_complete(self._testPipeLoggerToPipe(test_string, loop=loop))
+   self.assertEqual(test_string, output,
+   "x = %s, len(output) = %s" % (x, len(output)))
diff --git a/lib/portage/util/_async/PipeLogger.py 
b/lib/portage/util/_async/PipeLogger.py
index a4258f350..ce8afb846 100644
--- a/lib/portage/util/_async/PipeLogger.py
+++ b/lib/portage/util/_async/PipeLogger.py
@@ -8,6 +8,10 @@ import sys
 
 import portage
 from portage import os, _encodings, _unicode_encode
+from portage.util.futures import asyncio
+from portage.util.futures._asyncio.streams import _writer
+from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures.unix_events import _set_nonblocking
 from _emerge.AbstractPollTask import AbstractPollTask
 
 class PipeLogger(AbstractPollTask):
@@ -21,13 +25,16 @@ class PipeLogger(AbstractPollTask):
"""
 
__slots__ = ("input_fd", "log_file_path", "stdout_fd") + \
-   ("_log_file", "_log_file_real")
+   ("_io_loop_task", "_log_file", "_log_file_nb", "_log_file_real")
 
def _start(self):
 
log_file_path = self.log_file_path
-   if log_file_path is not None:
-
+   if hasattr(log_file_path, 'write'):
+   self._log_file_nb = True
+   self._log_file = log_file_path
+   _set_nonblocking(self._log_file.fileno())
+   elif log_file_path is not None:
self._log_file = open(_unicode_encode(log_file_path,
encoding=_encodings['fs'], errors='strict'), 
mode='ab')
if log_file_path.endswith('.gz'):
@@ -40,9 +47,9 @@ class PipeLogger(AbstractPollTask):
mode=0o660)
 
if 

[gentoo-portage-dev] [PATCH 0/2] Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)

2020-06-19 Thread Zac Medico
This variable specifies a command that filters build log output to a
log file. The plan is to extend this to support a separate filter for
tty output in the future.

Previous versions of these patches were affected by bug 716636 which
was due to unsafe remove_reader and remove_writer calls in finally
clauses of the PipeLogger _io_loop coroutine. The remove_reader and
remove_writer calls are now skipped if the corresponding file object
has already been closed (which is normal if the coroutine has been
cancelled). Since this kind of bug is not easy to reproduce, currently
Rick Farina  is testing the patches in order to
verify that they do not trigger emerge hangs like bug 716636.

Bug: https://bugs.gentoo.org/709746
Bug: https://bugs.gentoo.org/716636

Zac Medico (2):
  PipeLogger: non-blocking write to pipe (bug 709746)
  Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)

 lib/_emerge/AbstractEbuildProcess.py  |   3 +-
 lib/_emerge/BinpkgFetcher.py  |   3 +-
 lib/_emerge/EbuildFetcher.py  |   3 +-
 lib/_emerge/EbuildPhase.py|  47 ++--
 lib/_emerge/SpawnProcess.py   |  58 +++---
 lib/portage/dbapi/_MergeProcess.py|   3 +-
 .../ebuild/_config/special_env_vars.py|   8 +-
 lib/portage/tests/process/test_PipeLogger.py  |  58 ++
 lib/portage/util/_async/BuildLogger.py| 109 ++
 lib/portage/util/_async/PipeLogger.py |  73 +---
 lib/portage/util/_async/SchedulerInterface.py |  32 -
 man/make.conf.5   |   7 +-
 12 files changed, 358 insertions(+), 46 deletions(-)
 create mode 100644 lib/portage/tests/process/test_PipeLogger.py
 create mode 100644 lib/portage/util/_async/BuildLogger.py

-- 
2.25.3




[gentoo-portage-dev] [PATCH 2/2] Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)

2020-06-19 Thread Zac Medico
This variable specifies a command that filters build log output to a
log file. The plan is to extend this to support a separate filter for
tty output in the future.

In order to enable the EbuildPhase class to write elog messages to
the build log with PORTAGE_LOG_FILTER_FILE_CMD support, convert its
_elog method to a coroutine, and add a SchedulerInterface async_output
method for it to use.

Use a new BuildLogger class to manage log output (with or without a
filter command), with compression support provided by PipeLogger.
BuildLogger has a stdin property which provides access to a writable
binary file stream (refers to a pipe) that log content is written to.

Bug: https://bugs.gentoo.org/709746
Signed-off-by: Zac Medico 
---
 lib/_emerge/AbstractEbuildProcess.py  |   3 +-
 lib/_emerge/BinpkgFetcher.py  |   3 +-
 lib/_emerge/EbuildFetcher.py  |   3 +-
 lib/_emerge/EbuildPhase.py|  47 ++--
 lib/_emerge/SpawnProcess.py   |  58 +++---
 lib/portage/dbapi/_MergeProcess.py|   3 +-
 .../ebuild/_config/special_env_vars.py|   8 +-
 lib/portage/util/_async/BuildLogger.py| 109 ++
 lib/portage/util/_async/SchedulerInterface.py |  32 -
 man/make.conf.5   |   7 +-
 10 files changed, 243 insertions(+), 30 deletions(-)
 create mode 100644 lib/portage/util/_async/BuildLogger.py

diff --git a/lib/_emerge/AbstractEbuildProcess.py 
b/lib/_emerge/AbstractEbuildProcess.py
index 1c1955cfe..ae1aae55f 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2019 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -196,6 +196,7 @@ class AbstractEbuildProcess(SpawnProcess):
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
 
+   self.log_filter_file = 
self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
try:
SpawnProcess._start(self)
finally:
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 36d027de3..2e5861cc1 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -158,6 +158,7 @@ class _BinpkgFetcherProcess(SpawnProcess):
self.env = fetch_env
if settings.selinux_enabled():
self._selinux_type = settings["PORTAGE_FETCH_T"]
+   self.log_filter_file = 
settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
SpawnProcess._start(self)
 
def _pipe(self, fd_pipes):
diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py
index 1e40994fb..55349c33c 100644
--- a/lib/_emerge/EbuildFetcher.py
+++ b/lib/_emerge/EbuildFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import copy
@@ -225,6 +225,7 @@ class _EbuildFetcherProcess(ForkProcess):
settings["NOCOLOR"] = nocolor
 
self._settings = settings
+   self.log_filter_file = 
settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
ForkProcess._start(self)
 
# Free settings now since it's no longer needed in
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 477e0ba97..ddb3dc719 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -26,6 +26,8 @@ from portage.package.ebuild.prepare_build_dirs import 
(_prepare_workdir,
 from portage.util.futures.compat_coroutine import coroutine
 from portage.util import writemsg
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._async.BuildLogger import BuildLogger
+from portage.util.futures import asyncio
 from portage.util.futures.executor.fork import ForkExecutor
 
 try:
@@ -69,6 +71,11 @@ class EbuildPhase(CompositeTask):
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
 
def _start(self):
+   future = asyncio.ensure_future(self._async_start(), 
loop=self.scheduler)
+   self._start_task(AsyncTaskFuture(future=future), 
self._async_start_exit)
+
+   @coroutine
+   def _async_start(self):
 
need_builddir = self.phase not in 
EbuildProcess._phases_without_builddir
 
@@ -126,7 +133,7 @@ class EbuildPhase(CompositeTask):
# Force background=True for this header since it's 
intended
# for the log and it doesn't necessarily need to be 
visible
# elsewhere.
-