[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2024-02-27 Thread Zac Medico 
commit: e882b1e956d50808a0143875a8ca35f2fc21f368
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Feb 28 06:25:45 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Feb 28 06:25:45 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e882b1e9

UnshareNetTestCase: Initialize ABILITY_TO_UNSHARE in setUp

Initialize ABILITY_TO_UNSHARE in setUp so that _unshare_validate
uses the correct PORTAGE_MULTIPROCESSING_START_METHOD setup
from super().setUp(), eliminating messages like this from CI
runs for the multiprocessing start method spawn:


/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/popen_fork.py:66:
DeprecationWarning: This process (pid=2886) is multi-threaded, use of 
fork() may lead to deadlocks in the child.

The cause of these messages can be traced by patching python's
multiprocessing popen_fork.py like this:

--- /usr/lib/python3.12/multiprocessing/popen_fork.py
+++ /usr/lib/python3.12/multiprocessing/popen_fork.py
@@ -2,2 +2,3 @@
 import signal
+import traceback

@@ -62,2 +63,3 @@
 def _launch(self, process_obj):
+traceback.print_stack()
 code = 1

Fixes: 3110ec376cbc ("actions: Fix interaction between start-method and 
pytest-xdist")
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/tests/process/test_unshare_net.py | 18 +-
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/lib/portage/tests/process/test_unshare_net.py 
b/lib/portage/tests/process/test_unshare_net.py
index dabf15585f..ad3b288ef4 100644
--- a/lib/portage/tests/process/test_unshare_net.py
+++ b/lib/portage/tests/process/test_unshare_net.py
@@ -20,19 +20,27 @@ ping -c 1 -W 1 10.0.0.1 || exit 1
 ping -c 1 -W 1 ::1 || exit 1
 ping -c 1 -W 1 fd::1 || exit 1
 """
-ABILITY_TO_UNSHARE = portage.process._unshare_validate(CLONE_NEWNET)
 
 
 class UnshareNetTestCase(TestCase):
-@pytest.mark.skipif(
-ABILITY_TO_UNSHARE != 0,
-reason=f"Unable to unshare: {errno.errorcode.get(ABILITY_TO_UNSHARE, 
'?')}",
-)
+def setUp(self):
+"""
+Initialize ABILITY_TO_UNSHARE in setUp so that _unshare_validate
+uses the correct PORTAGE_MULTIPROCESSING_START_METHOD setup
+from super().setUp().
+"""
+super().setUp()
+self.ABILITY_TO_UNSHARE = 
portage.process._unshare_validate(CLONE_NEWNET)
+
 @pytest.mark.skipif(
 portage.process.find_binary("ping") is None, reason="ping not found"
 )
 @pytest.mark.skipif(platform.system() != "Linux", reason="not Linux")
 def testUnshareNet(self):
+if self.ABILITY_TO_UNSHARE != 0:
+pytest.skip(
+f"Unable to unshare: 
{errno.errorcode.get(self.ABILITY_TO_UNSHARE, '?')}"
+)
 env = os.environ.copy()
 env["IPV6"] = "1" if portage.process._has_ipv6() else ""
 self.assertEqual(

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2024-02-24 Thread Zac Medico 
commit: 9e84ef57ba747766c9147c1ac1b247faa1f05956
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Feb 25 00:31:13 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Feb 25 00:42:46 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9e84ef57

testSpawnReturnProcTerminate: Fix integer in spawn command argument

The invalid integer in the spawn command argument intermittently
triggered this error when SIGTERM did not arrive until after the
exec call:

- Captured stderr call -
Process ForkProcess-23:
Traceback (most recent call last):
  File "/home/runner/work/portage/portage/lib/portage/process.py", line 818, in 
_exec_wrapper
_exec(
  File "/home/runner/work/portage/portage/lib/portage/process.py", line 1068, 
in _exec
_exec2(
  File "/home/runner/work/portage/portage/lib/portage/process.py", line 1166, 
in _exec2
os.execve(binary, myargs, env)
TypeError: expected str, bytes or os.PathLike object, not int

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File 
"/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/process.py",
 line 314, in _bootstrap
self.run()
  File 
"/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/process.py",
 line 108, in run
self._target(*self._args, **self._kwargs)
  File 
"/home/runner/work/portage/portage/lib/portage/util/_async/ForkProcess.py", 
line 328, in _bootstrap
sys.exit(target(*(args or []), **(kwargs or {})))
 ^^^
  File "/home/runner/work/portage/portage/lib/portage/process.py", line 1441, 
in __call__
return self._target(*args, **kwargs)
   ^
  File "/home/runner/work/portage/portage/lib/portage/process.py", line 853, in 
_exec_wrapper
writemsg(f"{e}:\n   {' '.join(mycommand)}\n", noiselevel=-1)
 ^^^
TypeError: sequence item 1: expected str instance, int found

Bug: https://bugs.gentoo.org/916566#c20
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/tests/process/test_spawn_returnproc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/tests/process/test_spawn_returnproc.py 
b/lib/portage/tests/process/test_spawn_returnproc.py
index 6d823d9c3d..8fbf54d0d2 100644
--- a/lib/portage/tests/process/test_spawn_returnproc.py
+++ b/lib/portage/tests/process/test_spawn_returnproc.py
@@ -32,7 +32,7 @@ class SpawnReturnProcTestCase(TestCase):
 loop = global_event_loop()
 
 async def watch_pid():
-proc = spawn([sleep_binary, ], returnproc=True)
+proc = spawn([sleep_binary, ""], returnproc=True)
 proc.terminate()
 self.assertEqual(await proc.wait(), -signal.SIGTERM)

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2024-02-08 Thread Sam James 
commit: a308d831ceb1f1e7d0733700117cc18c8eca09c8
Author: Matoro Mahri  matoro  tk>
AuthorDate: Wed Jan 31 20:26:06 2024 +
Commit: Sam James  gentoo  org>
CommitDate: Fri Feb  9 07:08:22 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a308d831

tests: process: calculate size of E2BIG environment variable dynamically

The size of the command line required to trigger E2BIG response from the
kernel is defined as MAX_ARG_STRLEN, which is equal to 32 * PAGE_SIZE.
Therefore the minimum size required to trigger the expected error
response must be calculated dynamically based on PAGE_SIZE.

Bug: https://bugs.gentoo.org/923368
Signed-off-by: Matoro Mahri  matoro.tk>
Closes: https://github.com/gentoo/portage/pull/1246
Signed-off-by: Sam James  gentoo.org>

 lib/portage/tests/process/test_spawn_fail_e2big.py | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/portage/tests/process/test_spawn_fail_e2big.py 
b/lib/portage/tests/process/test_spawn_fail_e2big.py
index 7a00966302..abb1113fea 100644
--- a/lib/portage/tests/process/test_spawn_fail_e2big.py
+++ b/lib/portage/tests/process/test_spawn_fail_e2big.py
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import platform
+import resource
 
 import pytest
 
@@ -12,7 +13,9 @@ from portage.const import BASH_BINARY
 @pytest.mark.skipif(platform.system() != "Linux", reason="not Linux")
 def test_spawnE2big(capsys, tmp_path):
 env = dict()
-env["VERY_LARGE_ENV_VAR"] = "X" * 1024 * 256
+# Kernel MAX_ARG_STRLEN is defined as 32 * PAGE_SIZE
+max_arg_strlen_bytes = 32 * resource.getpagesize()
+env["VERY_LARGE_ENV_VAR"] = "X" * max_arg_strlen_bytes
 
 logfile = tmp_path / "logfile"
 echo_output = "Should never appear"
@@ -24,7 +27,7 @@ def test_spawnE2big(capsys, tmp_path):
 with open(logfile) as f:
 logfile_content = f.read()
 assert (
-"Largest environment variable: VERY_LARGE_ENV_VAR (262164 bytes)"
+f"Largest environment variable: VERY_LARGE_ENV_VAR 
({max_arg_strlen_bytes + 20} bytes)"
 in logfile_content
 )
 assert retval == 1

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2024-02-03 Thread Zac Medico 
commit: 7ea1175091886baa677d11290d4b725a64e68710
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Feb  3 19:36:13 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Feb  3 19:55:19 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7ea11750

Revert "testAsyncFunctionStdin: multiprocessing spawn compat"

This reverts commit 3b1234ba69a31709cd5aec1ae070901e3a28bb7c,
since ForkProcess now solves the problem by creating temporary
duplicate file descriptors.

Bug: https://bugs.gentoo.org/916601
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/tests/process/test_AsyncFunction.py | 19 ---
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/lib/portage/tests/process/test_AsyncFunction.py 
b/lib/portage/tests/process/test_AsyncFunction.py
index a056f268bd..eb426a5c02 100644
--- a/lib/portage/tests/process/test_AsyncFunction.py
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Gentoo Authors
+# Copyright 2020-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -44,20 +44,17 @@ class AsyncFunctionTestCase(TestCase):
 ),
 )
 reader.start()
-# For compatibility with the multiprocessing spawn start
-# method, we delay restoration of the stdin file descriptor,
-# since this file descriptor is sent to the subprocess
-# asynchronously.
-_set_nonblocking(pw.fileno())
-with open(pw.fileno(), mode="wb", buffering=0, closefd=False) as 
pipe_write:
-await _writer(pipe_write, test_string.encode("utf_8"))
-pw.close()
-self.assertEqual((await reader.async_wait()), os.EX_OK)
-self.assertEqual(reader.result, test_string)
 finally:
 os.dup2(stdin_backup, portage._get_stdin().fileno())
 os.close(stdin_backup)
 
+_set_nonblocking(pw.fileno())
+with open(pw.fileno(), mode="wb", buffering=0, closefd=False) as 
pipe_write:
+await _writer(pipe_write, test_string.encode("utf_8"))
+pw.close()
+self.assertEqual((await reader.async_wait()), os.EX_OK)
+self.assertEqual(reader.result, test_string)
+
 def testAsyncFunctionStdin(self):
 loop = asyncio._wrap_loop()
 loop.run_until_complete(self._testAsyncFunctionStdin(loop=loop))

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2023-11-02 Thread Zac Medico 
commit: 3b1234ba69a31709cd5aec1ae070901e3a28bb7c
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Nov  1 05:07:32 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Nov  2 14:56:37 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3b1234ba

testAsyncFunctionStdin: multiprocessing spawn compat

For compatibility with the multiprocessing spawn start
method, we delay restoration of the stdin file descriptor,
since this file descriptor is sent to the subprocess
asynchronously.

Signed-off-by: Zac Medico  gentoo.org>
Bug: https://bugs.gentoo.org/916601

 lib/portage/tests/process/test_AsyncFunction.py | 30 +++--
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/lib/portage/tests/process/test_AsyncFunction.py 
b/lib/portage/tests/process/test_AsyncFunction.py
index d02a3395c1..1faf8f49f7 100644
--- a/lib/portage/tests/process/test_AsyncFunction.py
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -42,21 +42,37 @@ class AsyncFunctionTestCase(TestCase):
 ),
 )
 reader.start()
+# For compatibility with the multiprocessing spawn start
+# method, we delay restoration of the stdin file descriptor,
+# since this file descriptor is sent to the subprocess
+# asynchronously.
+_set_nonblocking(pw.fileno())
+with open(pw.fileno(), mode="wb", buffering=0, closefd=False) as 
pipe_write:
+await _writer(pipe_write, test_string.encode("utf_8"))
+pw.close()
+self.assertEqual((await reader.async_wait()), os.EX_OK)
+self.assertEqual(reader.result, test_string)
 finally:
 os.dup2(stdin_backup, portage._get_stdin().fileno())
 os.close(stdin_backup)
 
-_set_nonblocking(pw.fileno())
-with open(pw.fileno(), mode="wb", buffering=0, closefd=False) as 
pipe_write:
-await _writer(pipe_write, test_string.encode("utf_8"))
-pw.close()
-self.assertEqual((await reader.async_wait()), os.EX_OK)
-self.assertEqual(reader.result, test_string)
-
 def testAsyncFunctionStdin(self):
 loop = asyncio._wrap_loop()
 loop.run_until_complete(self._testAsyncFunctionStdin(loop=loop))
 
+def testAsyncFunctionStdinSpawn(self):
+orig_start_method = multiprocessing.get_start_method()
+if orig_start_method == "spawn":
+self.skipTest("multiprocessing start method is already spawn")
+# NOTE: An attempt was made to use multiprocessing.get_context("spawn")
+# here, but it caused the python process to terminate unexpectedly
+# during a send_handle call.
+multiprocessing.set_start_method("spawn", force=True)
+try:
+self.testAsyncFunctionStdin()
+finally:
+multiprocessing.set_start_method(orig_start_method, force=True)
+
 @staticmethod
 def _test_getpid_fork(preexec_fn=None):
 """

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/, lib/portage/util/_async/, lib/_emerge/

2023-10-17 Thread Zac Medico 
commit: 767a746035960a211229fac33aabc042d30234b7
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Oct 17 01:54:10 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Oct 17 16:33:57 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=767a7460

ForkProcess: Support logging via multiprocessing.Pipe

Detect the multiprocessing spawn start method, and log via
multiprocessing.Pipe because file descriptor inheritance
via fd_pipes does not work with the spawn start method.

The included test cases set the multiprocessing start
method to spawn in a subprocess, in order to prevent
interference with the main process.

Bug: https://bugs.gentoo.org/915123
Signed-off-by: Zac Medico  gentoo.org>

 lib/_emerge/SpawnProcess.py |  9 +++-
 lib/portage/tests/process/meson.build   |  1 +
 lib/portage/tests/process/test_AsyncFunction.py | 23 +-
 lib/portage/tests/process/test_ForkProcess.py   | 39 +
 lib/portage/util/_async/ForkProcess.py  | 58 ++---
 5 files changed, 120 insertions(+), 10 deletions(-)

diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 0e27f29c43..b4eabd07ad 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -151,6 +151,12 @@ class SpawnProcess(SubProcess):
 if can_log and not self.background:
 stdout_fd = os.dup(fd_pipes_orig[1])
 
+self._start_main_task(
+master_fd, log_file_path=log_file_path, stdout_fd=stdout_fd
+)
+self._registered = True
+
+def _start_main_task(self, pr, log_file_path=None, stdout_fd=None):
 build_logger = BuildLogger(
 env=self.env,
 log_path=log_file_path,
@@ -162,14 +168,13 @@ class SpawnProcess(SubProcess):
 pipe_logger = PipeLogger(
 background=self.background,
 scheduler=self.scheduler,
-input_fd=master_fd,
+input_fd=pr,
 log_file_path=build_logger.stdin,
 stdout_fd=stdout_fd,
 )
 
 pipe_logger.start()
 
-self._registered = True
 self._main_task_cancel = functools.partial(
 self._main_cancel, build_logger, pipe_logger
 )

diff --git a/lib/portage/tests/process/meson.build 
b/lib/portage/tests/process/meson.build
index 1cef1cc4a2..b86fa10fb1 100644
--- a/lib/portage/tests/process/meson.build
+++ b/lib/portage/tests/process/meson.build
@@ -1,6 +1,7 @@
 py.install_sources(
 [
 'test_AsyncFunction.py',
+'test_ForkProcess.py',
 'test_PipeLogger.py',
 'test_PopenProcessBlockingIO.py',
 'test_PopenProcess.py',

diff --git a/lib/portage/tests/process/test_AsyncFunction.py 
b/lib/portage/tests/process/test_AsyncFunction.py
index 975b590e53..e30ff770b3 100644
--- a/lib/portage/tests/process/test_AsyncFunction.py
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -1,6 +1,7 @@
 # Copyright 2020-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+import functools
 import multiprocessing
 import sys
 
@@ -46,10 +47,12 @@ class AsyncFunctionTestCase(TestCase):
 loop.run_until_complete(self._testAsyncFunctionStdin(loop=loop))
 
 @staticmethod
-def _test_getpid_fork():
+def _test_getpid_fork(preexec_fn=None):
 """
 Verify that portage.getpid() cache is updated in a forked child 
process.
 """
+if preexec_fn is not None:
+preexec_fn()
 loop = asyncio._wrap_loop()
 proc = AsyncFunction(scheduler=loop, target=portage.getpid)
 proc.start()
@@ -59,6 +62,24 @@ class AsyncFunctionTestCase(TestCase):
 def test_getpid_fork(self):
 self.assertTrue(self._test_getpid_fork())
 
+def test_spawn_getpid(self):
+"""
+Test portage.getpid() with multiprocessing spawn start method.
+"""
+loop = asyncio._wrap_loop()
+proc = AsyncFunction(
+scheduler=loop,
+target=self._test_getpid_fork,
+kwargs=dict(
+preexec_fn=functools.partial(
+multiprocessing.set_start_method, "spawn", force=True
+)
+),
+)
+proc.start()
+self.assertEqual(proc.wait(), 0)
+self.assertTrue(proc.result)
+
 def test_getpid_double_fork(self):
 """
 Verify that portage.getpid() cache is updated correctly after

diff --git a/lib/portage/tests/process/test_ForkProcess.py 
b/lib/portage/tests/process/test_ForkProcess.py
new file mode 100644
index 00..c07c60e9c6
--- /dev/null
+++ b/lib/portage/tests/process/test_ForkProcess.py
@@ -0,0 +1,39 @@
+# Copyright 2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import functools
+import multiprocessing
+import tempfile
+
+from portage import os
+from portage.tests import TestCase
+from

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/, lib/portage/cache/

2023-10-15 Thread Zac Medico 
commit: 8835140f9da1f43fa0e7360aaf403bc635ab7398
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Oct 16 03:58:40 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Oct 16 04:41:04 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8835140f

slot_dict_class: Make instances picklable

This improves compatibility with the multiprocessing spawn
start method, by eliminating this error for MergeProcess:

AttributeError: Can't pickle local object 
'slot_dict_class..LocalSlotDict'

Bug: https://bugs.gentoo.org/915834
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/cache/mappings.py| 23 +
 lib/portage/tests/process/meson.build|  1 +
 lib/portage/tests/process/test_pickle.py | 43 
 3 files changed, 67 insertions(+)

diff --git a/lib/portage/cache/mappings.py b/lib/portage/cache/mappings.py
index 1aacad99a4..e7ee7df5cd 100644
--- a/lib/portage/cache/mappings.py
+++ b/lib/portage/cache/mappings.py
@@ -292,6 +292,16 @@ class _SlotDict:
 if kwargs:
 self.update(kwargs)
 
+def __reduce__(self):
+return _PickledSlotDict, (
+self._prefix,
+self.allowed_keys,
+dict(self),
+)
+
+def __eq__(self, other):
+return dict(self) == dict(other)
+
 def __iter__(self):
 for k, v in self.iteritems():
 yield k
@@ -417,6 +427,19 @@ class _SlotDict:
 values = itervalues
 
 
+class _PickledSlotDict(_SlotDict):
+"""
+Since LocalSlotDict instances are not directly picklable, this
+class exists as a way to express pickled LocalSlotDict instances,
+using a plain __dict__ instead of custom __slots__.
+"""
+
+def __init__(self, prefix, allowed_keys, *args, **kwargs):
+self._prefix = prefix
+self.allowed_keys = allowed_keys
+super().__init__(*args, **kwargs)
+
+
 _slot_dict_classes = weakref.WeakValueDictionary()
 
 

diff --git a/lib/portage/tests/process/meson.build 
b/lib/portage/tests/process/meson.build
index 1b34e57e33..1cef1cc4a2 100644
--- a/lib/portage/tests/process/meson.build
+++ b/lib/portage/tests/process/meson.build
@@ -4,6 +4,7 @@ py.install_sources(
 'test_PipeLogger.py',
 'test_PopenProcessBlockingIO.py',
 'test_PopenProcess.py',
+'test_pickle.py',
 'test_poll.py',
 'test_spawn_fail_e2big.py',
 'test_spawn_warn_large_env.py',

diff --git a/lib/portage/tests/process/test_pickle.py 
b/lib/portage/tests/process/test_pickle.py
new file mode 100644
index 00..9b7d9ef423
--- /dev/null
+++ b/lib/portage/tests/process/test_pickle.py
@@ -0,0 +1,43 @@
+# Copyright 2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import pickle
+
+from portage.tests import TestCase
+from _emerge.Package import _PackageMetadataWrapperBase
+from _emerge.FifoIpcDaemon import FifoIpcDaemon
+
+
+class PickleTestCase(TestCase):
+def test_PackageMetadataWrapperBase(self):
+"""
+Verify that instances of slot_dict_class, like
+PackageMetadataWrapperBase, are picklable for
+compatibility with the multiprocessing spawn
+start method.
+"""
+obj = _PackageMetadataWrapperBase(EAPI="8")
+self.assertEqual(obj["EAPI"], "8")
+serialized = pickle.dumps(obj)
+obj_copy = pickle.loads(serialized)
+self.assertEqual(len(obj_copy), len(obj))
+self.assertEqual(obj_copy["EAPI"], obj["EAPI"])
+self.assertEqual(obj_copy, obj)
+
+def test_FifoIpcDaemon_files_dict(self):
+"""
+Verify that FifoIpcDaemon._files_dict instances are picklable for
+compatibility with the multiprocessing spawn start method.
+"""
+obj = FifoIpcDaemon._files_dict(
+(k, "test-value") for k in FifoIpcDaemon._file_names
+)
+self.assertEqual(obj["pipe_in"], "test-value")
+# Attributes of same name exist because of slot_dict_class prefix="" 
argument.
+self.assertEqual(obj.pipe_in, obj["pipe_in"])
+serialized = pickle.dumps(obj)
+obj_copy = pickle.loads(serialized)
+self.assertEqual(len(obj_copy), len(obj))
+self.assertEqual(obj_copy["pipe_in"], obj["pipe_in"])
+self.assertEqual(obj_copy.pipe_in, obj["pipe_in"])
+self.assertEqual(obj_copy, obj)

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/, lib/portage/util/_async/

2023-10-03 Thread Zac Medico 
commit: 1c0b5ff6aabe99224b07f7fc2e09e4cb6c9c70c4
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Oct  3 20:33:18 2023 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Oct  4 00:47:01 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1c0b5ff6

AsyncFunction: Use multiprocessing.Pipe for compat with multiprocessing spawn

Since fd_pipes does not work with the multiprocessing spawn start
method, AsyncFunction should use multiprocessing.Pipe instead.

Bug: https://bugs.gentoo.org/915136
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/tests/process/test_AsyncFunction.py | 34 -
 lib/portage/util/_async/AsyncFunction.py| 15 +++
 2 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/lib/portage/tests/process/test_AsyncFunction.py 
b/lib/portage/tests/process/test_AsyncFunction.py
index 81b3f41fbf..975b590e53 100644
--- a/lib/portage/tests/process/test_AsyncFunction.py
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -1,6 +1,7 @@
-# Copyright 2020-2021 Gentoo Authors
+# Copyright 2020-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+import multiprocessing
 import sys
 
 import portage
@@ -14,22 +15,29 @@ from portage.util.futures.unix_events import 
_set_nonblocking
 
 class AsyncFunctionTestCase(TestCase):
 @staticmethod
-def _read_from_stdin(pw):
-os.close(pw)
+def _read_from_stdin(pr, pw):
+if pw is not None:
+os.close(pw)
+os.dup2(pr.fileno(), sys.stdin.fileno())
 return "".join(sys.stdin)
 
 async def _testAsyncFunctionStdin(self, loop):
 test_string = "1\n2\n3\n"
-pr, pw = os.pipe()
-fd_pipes = {0: pr}
+pr, pw = multiprocessing.Pipe(duplex=False)
 reader = AsyncFunction(
-scheduler=loop, fd_pipes=fd_pipes, target=self._read_from_stdin, 
args=(pw,)
+scheduler=loop,
+target=self._read_from_stdin,
+args=(
+pr,
+pw.fileno() if multiprocessing.get_start_method() == "fork" 
else None,
+),
 )
 reader.start()
-os.close(pr)
-_set_nonblocking(pw)
-with open(pw, mode="wb", buffering=0) as pipe_write:
+pr.close()
+_set_nonblocking(pw.fileno())
+with open(pw.fileno(), mode="wb", buffering=0, closefd=False) as 
pipe_write:
 await _writer(pipe_write, test_string.encode("utf_8"))
+pw.close()
 self.assertEqual((await reader.async_wait()), os.EX_OK)
 self.assertEqual(reader.result, test_string)
 
@@ -37,7 +45,8 @@ class AsyncFunctionTestCase(TestCase):
 loop = asyncio._wrap_loop()
 loop.run_until_complete(self._testAsyncFunctionStdin(loop=loop))
 
-def _test_getpid_fork(self):
+@staticmethod
+def _test_getpid_fork():
 """
 Verify that portage.getpid() cache is updated in a forked child 
process.
 """
@@ -45,10 +54,10 @@ class AsyncFunctionTestCase(TestCase):
 proc = AsyncFunction(scheduler=loop, target=portage.getpid)
 proc.start()
 proc.wait()
-self.assertEqual(proc.pid, proc.result)
+return proc.pid == proc.result
 
 def test_getpid_fork(self):
-self._test_getpid_fork()
+self.assertTrue(self._test_getpid_fork())
 
 def test_getpid_double_fork(self):
 """
@@ -59,3 +68,4 @@ class AsyncFunctionTestCase(TestCase):
 proc = AsyncFunction(scheduler=loop, target=self._test_getpid_fork)
 proc.start()
 self.assertEqual(proc.wait(), 0)
+self.assertTrue(proc.result)

diff --git a/lib/portage/util/_async/AsyncFunction.py 
b/lib/portage/util/_async/AsyncFunction.py
index e13daaebb0..6f55aba565 100644
--- a/lib/portage/util/_async/AsyncFunction.py
+++ b/lib/portage/util/_async/AsyncFunction.py
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
+import multiprocessing
 import pickle
 import traceback
 
@@ -23,9 +24,7 @@ class AsyncFunction(ForkProcess):
 )
 
 def _start(self):
-pr, pw = os.pipe()
-self.fd_pipes = {} if self.fd_pipes is None else self.fd_pipes
-self.fd_pipes[pw] = pw
+pr, pw = multiprocessing.Pipe(duplex=False)
 self._async_func_reader = PipeReader(
 input_files={"input": pr}, scheduler=self.scheduler
 )
@@ -34,13 +33,15 @@ class AsyncFunction(ForkProcess):
 # args and kwargs are passed as additional args by 
ForkProcess._bootstrap.
 self.target = functools.partial(self._target_wrapper, pw, self.target)
 ForkProcess._start(self)
-os.close(pw)
+pw.close()
 
 @staticmethod
 def _target_wrapper(pw, target, *args, **kwargs):
 try:
 result = target(*args, **kwargs)
-os.write(pw, pickle.dumps(result))
+result_bytes =

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2023-05-26 Thread Sam James 
commit: 3db3c517d7e7635f3507f1e7f921dcc9fdd33c9a
Author: David Palao  gmail  com>
AuthorDate: Mon May  1 09:18:48 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Fri May 26 15:44:36 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3db3c517

tests: process: order of skipif conditions changed

...to agree with the original, checking if:

1. system is not Linux
2. ping is missing
3. portage.process._unshare_validate returns error

Signed-off-by: David Palao  gmail.com>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/tests/process/test_unshare_net.py | 10 --
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/portage/tests/process/test_unshare_net.py 
b/lib/portage/tests/process/test_unshare_net.py
index 4c70465f8..7910e1553 100644
--- a/lib/portage/tests/process/test_unshare_net.py
+++ b/lib/portage/tests/process/test_unshare_net.py
@@ -13,7 +13,6 @@ from portage.tests import TestCase
 
 
 CLONE_NEWNET = 0x4000
-
 UNSHARE_NET_TEST_SCRIPT = """
 ping -c 1 -W 1 127.0.0.1 || exit 1
 ping -c 1 -W 1 10.0.0.1 || exit 1
@@ -21,19 +20,18 @@ ping -c 1 -W 1 10.0.0.1 || exit 1
 ping -c 1 -W 1 ::1 || exit 1
 ping -c 1 -W 1 fd::1 || exit 1
 """
-
 ABILITY_TO_UNSHARE = portage.process._unshare_validate(CLONE_NEWNET)
 
 
 class UnshareNetTestCase(TestCase):
-@pytest.mark.skipif(platform.system() != "Linux", reason="not Linux")
-@pytest.mark.skipif(
-portage.process.find_binary("ping") is None, reason="ping not found"
-)
 @pytest.mark.skipif(
 ABILITY_TO_UNSHARE != 0,
 reason=f"Unable to unshare: {errno.errorcode.get(ABILITY_TO_UNSHARE, 
'?')}",
 )
+@pytest.mark.skipif(
+portage.process.find_binary("ping") is None, reason="ping not found"
+)
+@pytest.mark.skipif(platform.system() != "Linux", reason="not Linux")
 def testUnshareNet(self):
 env = os.environ.copy()
 env["IPV6"] = "1" if portage.process._has_ipv6() else ""

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2023-05-26 Thread Sam James 
commit: f027096346460cb031c2cb6c66e494b79d1b4433
Author: David Palao  gmail  com>
AuthorDate: Fri May  5 15:38:54 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Fri May 26 15:44:37 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f0270963

tests: process: port to pytest test about spawning with large env var

Signed-off-by: David Palao  gmail.com>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/tests/process/test_spawn_fail_e2big.py | 50 +-
 1 file changed, 20 insertions(+), 30 deletions(-)

diff --git a/lib/portage/tests/process/test_spawn_fail_e2big.py 
b/lib/portage/tests/process/test_spawn_fail_e2big.py
index d3be51670..7a0096630 100644
--- a/lib/portage/tests/process/test_spawn_fail_e2big.py
+++ b/lib/portage/tests/process/test_spawn_fail_e2big.py
@@ -2,39 +2,29 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import platform
-import tempfile
 
-from pathlib import Path
+import pytest
 
 import portage.process
-
-from portage import shutil
 from portage.const import BASH_BINARY
-from portage.tests import TestCase
-
-
-class SpawnE2bigTestCase(TestCase):
-def testSpawnE2big(self):
-if platform.system() != "Linux":
-self.skipTest("not Linux")
-
-env = dict()
-env["VERY_LARGE_ENV_VAR"] = "X" * 1024 * 256
 
-tmpdir = tempfile.mkdtemp()
-try:
-logfile = tmpdir / Path("logfile")
-echo_output = "Should never appear"
-retval = portage.process.spawn(
-[BASH_BINARY, "-c", "echo", echo_output], env=env, 
logfile=logfile
-)
 
-with open(logfile) as f:
-logfile_content = f.read()
-self.assertIn(
-"Largest environment variable: VERY_LARGE_ENV_VAR (262164 
bytes)",
-logfile_content,
-)
-self.assertEqual(retval, 1)
-finally:
-shutil.rmtree(tmpdir)
+@pytest.mark.skipif(platform.system() != "Linux", reason="not Linux")
+def test_spawnE2big(capsys, tmp_path):
+env = dict()
+env["VERY_LARGE_ENV_VAR"] = "X" * 1024 * 256
+
+logfile = tmp_path / "logfile"
+echo_output = "Should never appear"
+with capsys.disabled():
+retval = portage.process.spawn(
+[BASH_BINARY, "-c", "echo", echo_output], env=env, logfile=logfile
+)
+
+with open(logfile) as f:
+logfile_content = f.read()
+assert (
+"Largest environment variable: VERY_LARGE_ENV_VAR (262164 bytes)"
+in logfile_content
+)
+assert retval == 1

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2023-05-26 Thread Sam James 
commit: c36ef773b5b130ef8ee2219c5a371877be1bd41f
Author: David Palao  gmail  com>
AuthorDate: Fri May 19 12:45:13 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Fri May 26 15:44:37 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c36ef773

tests: process: port PipeReaderArrayTestCase to pytest

...since in some cases will fail. But it only happens
*sometimes*.

Signed-off-by: David Palao  gmail.com>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/tests/process/test_poll.py | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/portage/tests/process/test_poll.py 
b/lib/portage/tests/process/test_poll.py
index 284dd168e..371dd1906 100644
--- a/lib/portage/tests/process/test_poll.py
+++ b/lib/portage/tests/process/test_poll.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2020 Gentoo Authors
+# Copyright 1998-2020, 2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -7,6 +7,8 @@ import shutil
 import socket
 import tempfile
 
+import pytest
+
 from portage import os
 from portage.tests import TestCase
 from portage.util._eventloop.global_event_loop import global_event_loop
@@ -109,6 +111,7 @@ class PipeReaderTestCase(TestCase):
 cleanup()
 
 
+@pytest.mark.xfail()  # This fails sometimes, that's the reason of xfail here
 class PipeReaderArrayTestCase(PipeReaderTestCase):
 _use_array = True
 # sleep allows reliable triggering of the failure mode on fast computers

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/, lib/portage/tests/resolver/, ...

2023-05-26 Thread Sam James 
commit: 9b3a1ae4426fab150aa240da12ba60c1d5bc10da
Author: David Palao  gmail  com>
AuthorDate: Fri May 19 12:51:57 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Fri May 26 15:44:37 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9b3a1ae4

tests: adjusted the copyright years in some files

Signed-off-by: David Palao  gmail.com>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/tests/emerge/test_config_protect.py| 2 +-
 lib/portage/tests/emerge/test_emerge_blocker_file_collision.py | 2 +-
 lib/portage/tests/emerge/test_emerge_slot_abi.py   | 2 +-
 lib/portage/tests/emerge/test_simple.py| 2 +-
 lib/portage/tests/process/test_unshare_net.py  | 2 +-
 lib/portage/tests/resolver/test_autounmask_multilib_use.py | 2 +-
 lib/portage/tests/resolver/test_or_choices.py  | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/portage/tests/emerge/test_config_protect.py 
b/lib/portage/tests/emerge/test_config_protect.py
index 0b8308a34..97d0c236f 100644
--- a/lib/portage/tests/emerge/test_config_protect.py
+++ b/lib/portage/tests/emerge/test_config_protect.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2015 Gentoo Foundation
+# Copyright 2014-2015, 2023 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from functools import partial

diff --git a/lib/portage/tests/emerge/test_emerge_blocker_file_collision.py 
b/lib/portage/tests/emerge/test_emerge_blocker_file_collision.py
index a05fc47a9..d21287401 100644
--- a/lib/portage/tests/emerge/test_emerge_blocker_file_collision.py
+++ b/lib/portage/tests/emerge/test_emerge_blocker_file_collision.py
@@ -1,4 +1,4 @@
-# Copyright 2016 Gentoo Foundation
+# Copyright 2016, 2023 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import subprocess

diff --git a/lib/portage/tests/emerge/test_emerge_slot_abi.py 
b/lib/portage/tests/emerge/test_emerge_slot_abi.py
index 3fc13e08f..51cd3c43c 100644
--- a/lib/portage/tests/emerge/test_emerge_slot_abi.py
+++ b/lib/portage/tests/emerge/test_emerge_slot_abi.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2019 Gentoo Authors
+# Copyright 2012-2019, 2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import subprocess

diff --git a/lib/portage/tests/emerge/test_simple.py 
b/lib/portage/tests/emerge/test_simple.py
index 86c5dccc4..50b7eee31 100644
--- a/lib/portage/tests/emerge/test_simple.py
+++ b/lib/portage/tests/emerge/test_simple.py
@@ -1,4 +1,4 @@
-# Copyright 2011-2021 Gentoo Authors
+# Copyright 2011-2021, 2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import argparse

diff --git a/lib/portage/tests/process/test_unshare_net.py 
b/lib/portage/tests/process/test_unshare_net.py
index da5761cd1..e17357f4d 100644
--- a/lib/portage/tests/process/test_unshare_net.py
+++ b/lib/portage/tests/process/test_unshare_net.py
@@ -1,4 +1,4 @@
-# Copyright 2019 Gentoo Authors
+# Copyright 2019, 2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno

diff --git a/lib/portage/tests/resolver/test_autounmask_multilib_use.py 
b/lib/portage/tests/resolver/test_autounmask_multilib_use.py
index 9be0f09cf..3abbebbd5 100644
--- a/lib/portage/tests/resolver/test_autounmask_multilib_use.py
+++ b/lib/portage/tests/resolver/test_autounmask_multilib_use.py
@@ -1,4 +1,4 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013, 2023 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import pytest

diff --git a/lib/portage/tests/resolver/test_or_choices.py 
b/lib/portage/tests/resolver/test_or_choices.py
index 19ecfe31c..33707c71a 100644
--- a/lib/portage/tests/resolver/test_or_choices.py
+++ b/lib/portage/tests/resolver/test_or_choices.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2020, 2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import itertools

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2023-05-26 Thread Sam James 
commit: 68eec067946997d64462a9d5fe188d1d830ea7a0
Author: David Palao  gmail  com>
AuthorDate: Mon May  1 08:50:15 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Fri May 26 15:44:36 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=68eec067

tests: process: port testUnshareNet to pytest

...using skipif, instead of the old addPortageSkip mechanism.

Signed-off-by: David Palao  gmail.com>
Signed-off-by: Sam James  gentoo.org>

 lib/portage/tests/process/test_unshare_net.py | 22 +-
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/lib/portage/tests/process/test_unshare_net.py 
b/lib/portage/tests/process/test_unshare_net.py
index 6951faab6..4c70465f8 100644
--- a/lib/portage/tests/process/test_unshare_net.py
+++ b/lib/portage/tests/process/test_unshare_net.py
@@ -5,10 +5,13 @@ import errno
 import os
 import platform
 
+import pytest
+
 import portage.process
 from portage.const import BASH_BINARY
 from portage.tests import TestCase
 
+
 CLONE_NEWNET = 0x4000
 
 UNSHARE_NET_TEST_SCRIPT = """
@@ -19,18 +22,19 @@ ping -c 1 -W 1 ::1 || exit 1
 ping -c 1 -W 1 fd::1 || exit 1
 """
 
+ABILITY_TO_UNSHARE = portage.process._unshare_validate(CLONE_NEWNET)
+
 
 class UnshareNetTestCase(TestCase):
+@pytest.mark.skipif(platform.system() != "Linux", reason="not Linux")
+@pytest.mark.skipif(
+portage.process.find_binary("ping") is None, reason="ping not found"
+)
+@pytest.mark.skipif(
+ABILITY_TO_UNSHARE != 0,
+reason=f"Unable to unshare: {errno.errorcode.get(ABILITY_TO_UNSHARE, 
'?')}",
+)
 def testUnshareNet(self):
-if platform.system() != "Linux":
-self.skipTest("not Linux")
-if portage.process.find_binary("ping") is None:
-self.skipTest("ping not found")
-
-errno_value = portage.process._unshare_validate(CLONE_NEWNET)
-if errno_value != 0:
-self.skipTest(f"Unable to unshare: 
{errno.errorcode.get(errno_value, '?')}")
-
 env = os.environ.copy()
 env["IPV6"] = "1" if portage.process._has_ipv6() else ""
 self.assertEqual(

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/, lib/portage/package/ebuild/, /, man/, lib/portage/

2023-03-11 Thread Sam James 
commit: 7c5a083cc27fdc8a80630f18f954532d561d
Author: Florian Schmaus  gentoo  org>
AuthorDate: Fri Feb 17 09:04:03 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Sat Mar 11 18:52:27 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c5a0866

Add FEATURE="warn-on-large-env": warn on large child process environments

This adds a new feature to portage which issues a warning if portage
is about to execute a new child process with a large environment.

Signed-off-by: Florian Schmaus  gentoo.org>
Closes: https://github.com/gentoo/portage/pull/963
Signed-off-by: Sam James  gentoo.org>

 NEWS   |  3 +
 lib/portage/const.py   |  1 +
 lib/portage/package/ebuild/doebuild.py |  1 +
 lib/portage/process.py | 69 --
 .../tests/process/test_spawn_warn_large_env.py | 46 +++
 man/make.conf.5|  3 +
 6 files changed, 106 insertions(+), 17 deletions(-)

diff --git a/NEWS b/NEWS
index e67e488b6..a6daf1138 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,6 @@
+* New portage FEATURE warn-on-large-env, to emit a warning if portage
+  executes an ebuild-related child process with a large environment.
+
 portage-3.0.45.2 (2023-03-04)
 --
 

diff --git a/lib/portage/const.py b/lib/portage/const.py
index 99206fe2c..10a208ceb 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -213,6 +213,7 @@ SUPPORTED_FEATURES = frozenset(
 "userpriv",
 "usersandbox",
 "usersync",
+"warn-on-large-env",
 "webrsync-gpg",
 "xattr",
 )

diff --git a/lib/portage/package/ebuild/doebuild.py 
b/lib/portage/package/ebuild/doebuild.py
index 2226812e3..380f8f98d 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -209,6 +209,7 @@ def _doebuild_spawn(phase, settings, actionmap=None, 
**kwargs):
 kwargs["pidns"] = (
 "pid-sandbox" in settings.features and phase not in _global_pid_phases
 )
+kwargs["warn_on_large_env"] = "warn-on-large-env" in settings.features
 
 if phase == "depend":
 kwargs["droppriv"] = "userpriv" in settings.features

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 94850b6e7..4a1baedc6 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -6,6 +6,7 @@
 import atexit
 import errno
 import fcntl
+import logging
 import multiprocessing
 import platform
 import signal
@@ -15,6 +16,9 @@ import sys
 import traceback
 import os as _os
 
+from dataclasses import dataclass
+from functools import lru_cache
+
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
@@ -22,7 +26,7 @@ import portage
 
 portage.proxy.lazyimport.lazyimport(
 globals(),
-"portage.util:dump_traceback,writemsg",
+"portage.util:dump_traceback,writemsg,writemsg_level",
 )
 
 from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
@@ -241,6 +245,37 @@ def cleanup():
 pass
 
 
+@dataclass(frozen=True)
+class EnvStats:
+env_size: int
+env_largest_name: str
+env_largest_size: int
+
+
+def calc_env_stats(env) -> EnvStats:
+@lru_cache(1024)
+def encoded_length(s):
+return len(os.fsencode(s))
+
+env_size = 0
+env_largest_name = None
+env_largest_size = 0
+for env_name, env_value in env.items():
+env_name_size = encoded_length(env_name)
+env_value_size = encoded_length(env_value)
+# Add two for '=' and the terminating null byte.
+total_size = env_name_size + env_value_size + 2
+if total_size > env_largest_size:
+env_largest_name = env_name
+env_largest_size = total_size
+env_size += total_size
+
+return EnvStats(env_size, env_largest_name, env_largest_size)
+
+
+env_too_large_warnings = 0
+
+
 def spawn(
 mycommand,
 env=None,
@@ -261,6 +296,7 @@ def spawn(
 unshare_mount=False,
 unshare_pid=False,
 cgroup=None,
+warn_on_large_env=False,
 ):
 """
 Spawns a given command.
@@ -322,6 +358,18 @@ def spawn(
 
 env = os.environ if env is None else env
 
+env_stats = None
+if warn_on_large_env:
+env_stats = calc_env_stats(env)
+
+global env_too_large_warnings
+if env_stats.env_size > 1024 * 96 and env_too_large_warnings < 3:
+env_too_large_warnings += 1
+writemsg_level(
+f"WARNING: New process environment is large, executing 
{mycommand} may fail. Size: {env_stats.env_size} bytes. Largest environment 
variable: {env_stats.env_largest_name} ({env_stats.env_largest_size} bytes)",
+logging.WARNING,
+)
+
 # If an absolute path to an executable file isn't given
 # search for it unless we've been told not to.
 binary = mycommand[0]
@@ -445,24 +493,11 @@ def

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2021-01-18 Thread Zac Medico 
commit: b7469f2f9777e818e9c0f3fa46e90c3f7f1c904a
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Jan 18 11:49:28 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Jan 18 11:50:10 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7469f2f

AsyncFunctionTestCase: Use async and await syntax

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/tests/process/test_AsyncFunction.py | 10 --
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/portage/tests/process/test_AsyncFunction.py 
b/lib/portage/tests/process/test_AsyncFunction.py
index b3f80b8ac..ce4b2a93b 100644
--- a/lib/portage/tests/process/test_AsyncFunction.py
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Gentoo Authors
+# Copyright 2020-2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import sys
@@ -9,7 +9,6 @@ from portage.tests import TestCase
 from portage.util._async.AsyncFunction import AsyncFunction
 from portage.util.futures import asyncio
 from portage.util.futures._asyncio.streams import _writer
-from portage.util.futures.compat_coroutine import coroutine
 from portage.util.futures.unix_events import _set_nonblocking
 
 
@@ -20,8 +19,7 @@ class AsyncFunctionTestCase(TestCase):
os.close(pw)
return ''.join(sys.stdin)
 
-   @coroutine
-   def _testAsyncFunctionStdin(self, loop=None):
+   async def _testAsyncFunctionStdin(self, loop):
test_string = '1\n2\n3\n'
pr, pw = os.pipe()
fd_pipes = {0:pr}
@@ -30,8 +28,8 @@ class AsyncFunctionTestCase(TestCase):
os.close(pr)
_set_nonblocking(pw)
with open(pw, mode='wb', buffering=0) as pipe_write:
-   yield _writer(pipe_write, test_string.encode('utf_8'), 
loop=loop)
-   self.assertEqual((yield reader.async_wait()), os.EX_OK)
+   await _writer(pipe_write, test_string.encode('utf_8'))
+   self.assertEqual((await reader.async_wait()), os.EX_OK)
self.assertEqual(reader.result, test_string)
 
def testAsyncFunctionStdin(self):

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2021-01-18 Thread Zac Medico 
commit: b3b9acc13c432ccd1610a442d57257dfac763cc4
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Jan 18 09:43:24 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Jan 18 11:18:43 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b3b9acc1

PipeLoggerTestCase: Use async and await syntax

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/tests/process/test_PipeLogger.py | 22 ++
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/lib/portage/tests/process/test_PipeLogger.py 
b/lib/portage/tests/process/test_PipeLogger.py
index acc3b8af9..48e198bc5 100644
--- a/lib/portage/tests/process/test_PipeLogger.py
+++ b/lib/portage/tests/process/test_PipeLogger.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Gentoo Authors
+# Copyright 2020-2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from portage import os
@@ -6,14 +6,12 @@ from portage.tests import TestCase
 from portage.util._async.PipeLogger import PipeLogger
 from portage.util.futures import asyncio
 from portage.util.futures._asyncio.streams import _reader, _writer
-from portage.util.futures.compat_coroutine import coroutine, coroutine_return
 from portage.util.futures.unix_events import _set_nonblocking
 
 
 class PipeLoggerTestCase(TestCase):
 
-   @coroutine
-   def _testPipeLoggerToPipe(self, test_string, loop=None):
+   async def _testPipeLoggerToPipe(self, test_string, loop):
"""
Test PipeLogger writing to a pipe connected to a PipeReader.
This verifies that PipeLogger does not deadlock when writing
@@ -24,7 +22,7 @@ class PipeLoggerTestCase(TestCase):
input_fd, writer_pipe = os.pipe()
_set_nonblocking(writer_pipe)
writer_pipe = os.fdopen(writer_pipe, 'wb', 0)
-   writer = asyncio.ensure_future(_writer(writer_pipe, 
test_string.encode('ascii'), loop=loop), loop=loop)
+   writer = asyncio.ensure_future(_writer(writer_pipe, 
test_string.encode('ascii')))
writer.add_done_callback(lambda writer: writer_pipe.close())
 
pr, pw = os.pipe()
@@ -37,22 +35,22 @@ class PipeLoggerTestCase(TestCase):
 
# Before starting the reader, wait here for a moment, in order
# to exercise PipeLogger's handling of EAGAIN during write.
-   yield asyncio.wait([writer], timeout=0.01, loop=loop)
+   await asyncio.wait([writer], timeout=0.01)
 
-   reader = _reader(pr, loop=loop)
-   yield writer
-   content = yield reader
-   yield consumer.async_wait()
+   reader = _reader(pr)
+   await writer
+   content = await reader
+   await consumer.async_wait()
 
self.assertEqual(consumer.returncode, os.EX_OK)
 
-   coroutine_return(content.decode('ascii', 'replace'))
+   return content.decode('ascii', 'replace')
 
def testPipeLogger(self):
loop = asyncio._wrap_loop()
 
for x in (1, 2, 5, 6, 7, 8, 2**5, 2**10, 2**12, 2**13, 2**14, 
2**17, 2**17 + 1):
test_string = x * "a"
-   output = 
loop.run_until_complete(self._testPipeLoggerToPipe(test_string, loop=loop))
+   output = 
loop.run_until_complete(self._testPipeLoggerToPipe(test_string, loop))
self.assertEqual(test_string, output,
"x = %s, len(output) = %s" % (x, len(output)))

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/, lib/portage/util/futures/_asyncio/, ...

2020-08-18 Thread Zac Medico 
commit: dc7919541712d846574e6b7d672a3bed0ca7ef1a
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Aug 18 06:31:54 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Aug 19 04:01:46 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dc791954

coroutine: use explicit loop parameter (bug 737698)

In order to support local event loops within API functions
like doebuild, use an explicit loop parameter when calling a
coroutine. Internal code will now raise an AssertionError if
the loop parameter is omitted for a coroutine, but API
consumers may omit it.

Bug: https://bugs.gentoo.org/737698
Signed-off-by: Zac Medico  gentoo.org>

 lib/_emerge/Binpkg.py  |  8 ++--
 lib/_emerge/EbuildPhase.py | 16 
 lib/_emerge/Scheduler.py   |  4 +-
 lib/_emerge/SequentialTaskQueue.py |  4 +-
 lib/_emerge/SpawnProcess.py|  5 ++-
 lib/portage/dbapi/bintree.py   | 12 +++---
 lib/portage/dbapi/vartree.py   |  8 ++--
 .../repository/storage/hardlink_quarantine.py  | 26 ++---
 lib/portage/repository/storage/hardlink_rcu.py | 34 
 lib/portage/repository/storage/inplace.py  | 10 ++---
 lib/portage/repository/storage/interface.py| 10 ++---
 lib/portage/sync/syncbase.py   |  2 +-
 lib/portage/tests/dbapi/test_auxdb.py  |  9 +++--
 lib/portage/tests/emerge/test_simple.py|  6 +--
 lib/portage/tests/process/test_AsyncFunction.py|  4 +-
 lib/portage/tests/process/test_PipeLogger.py   |  2 +-
 .../util/futures/asyncio/test_child_watcher.py |  4 +-
 .../tests/util/futures/test_compat_coroutine.py| 45 --
 lib/portage/tests/util/test_socks5.py  |  2 +-
 lib/portage/util/_async/BuildLogger.py |  4 +-
 lib/portage/util/_async/ForkProcess.py |  6 +--
 lib/portage/util/_async/PipeLogger.py  |  4 +-
 lib/portage/util/_async/SchedulerInterface.py  |  4 +-
 lib/portage/util/futures/_asyncio/process.py   | 16 
 lib/portage/util/futures/_sync_decorator.py|  3 +-
 lib/portage/util/futures/compat_coroutine.py   |  7 +++-
 lib/portage/util/socks5.py |  4 +-
 repoman/lib/repoman/modules/scan/depend/profile.py |  4 +-
 28 files changed, 138 insertions(+), 125 deletions(-)

diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index b5a69f8e7..9d2909d42 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -250,11 +250,11 @@ class Binpkg(CompositeTask):
return
 
self._start_task(
-   AsyncTaskFuture(future=self._unpack_metadata()),
+   
AsyncTaskFuture(future=self._unpack_metadata(loop=self.scheduler)),
self._unpack_metadata_exit)
 
@coroutine
-   def _unpack_metadata(self):
+   def _unpack_metadata(self, loop=None):
 
dir_path = self.settings['PORTAGE_BUILDDIR']
 
@@ -271,7 +271,7 @@ class Binpkg(CompositeTask):
portage.prepare_build_dirs(self.settings["ROOT"], 
self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
 
-   yield self._bintree.dbapi.unpack_metadata(self.settings, infloc)
+   yield self._bintree.dbapi.unpack_metadata(self.settings, 
infloc, loop=self.scheduler)
check_missing_metadata = ("CATEGORY", "PF")
for k, v in zip(check_missing_metadata,
self._bintree.dbapi.aux_get(self.pkg.cpv, 
check_missing_metadata)):
@@ -333,7 +333,7 @@ class Binpkg(CompositeTask):
self._start_task(

AsyncTaskFuture(future=self._bintree.dbapi.unpack_contents(
self.settings,
-   self._image_dir)),
+   self._image_dir, loop=self.scheduler)),
self._unpack_contents_exit)
 
def _unpack_contents_exit(self, unpack_contents):

diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 4bc2749bd..e4c0428a6 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -70,11 +70,11 @@ class EbuildPhase(CompositeTask):
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
 
def _start(self):
-   future = asyncio.ensure_future(self._async_start(), 
loop=self.scheduler)
+   future = 
asyncio.ensure_future(self._async_start(loop=self.scheduler), 
loop=self.scheduler)
self._start_task(AsyncTaskFuture(future=future), 
self._async_start_exit)
 
@coroutine
-   def _async_start(self):
+   def _async_start(self, loop=None):
 
need_builddir = self.phase not in 
EbuildProcess._phases_without_builddir

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/, lib/portage/util/_async/

2020-07-18 Thread Zac Medico 
commit: 859fe1a14214fb1a188cc0b49f8683a94cd04011
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Jul 19 03:57:12 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Jul 19 04:20:51 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=859fe1a1

AsyncFunction: stdin support

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/tests/process/test_AsyncFunction.py | 38 +
 lib/portage/util/_async/AsyncFunction.py|  4 +--
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/lib/portage/tests/process/test_AsyncFunction.py 
b/lib/portage/tests/process/test_AsyncFunction.py
new file mode 100644
index 0..55857026d
--- /dev/null
+++ b/lib/portage/tests/process/test_AsyncFunction.py
@@ -0,0 +1,38 @@
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import sys
+
+from portage import os
+from portage.tests import TestCase
+from portage.util._async.AsyncFunction import AsyncFunction
+from portage.util.futures import asyncio
+from portage.util.futures._asyncio.streams import _writer
+from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures.unix_events import _set_nonblocking
+
+
+class AsyncFunctionTestCase(TestCase):
+
+   @staticmethod
+   def _read_from_stdin(pw):
+   os.close(pw)
+   return ''.join(sys.stdin)
+
+   @coroutine
+   def _testAsyncFunctionStdin(self, loop):
+   test_string = '1\n2\n3\n'
+   pr, pw = os.pipe()
+   fd_pipes = {0:pr}
+   reader = AsyncFunction(scheduler=loop, fd_pipes=fd_pipes, 
target=self._read_from_stdin, args=(pw,))
+   reader.start()
+   os.close(pr)
+   _set_nonblocking(pw)
+   with open(pw, mode='wb', buffering=0) as pipe_write:
+   yield _writer(pipe_write, test_string.encode('utf_8'), 
loop=loop)
+   self.assertEqual((yield reader.async_wait()), os.EX_OK)
+   self.assertEqual(reader.result, test_string)
+
+   def testAsyncFunctionStdin(self):
+   loop = asyncio._wrap_loop()
+   loop.run_until_complete(self._testAsyncFunctionStdin(loop))

diff --git a/lib/portage/util/_async/AsyncFunction.py 
b/lib/portage/util/_async/AsyncFunction.py
index 1dffa36cc..db77a6f43 100644
--- a/lib/portage/util/_async/AsyncFunction.py
+++ b/lib/portage/util/_async/AsyncFunction.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Gentoo Foundation
+# Copyright 2015-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import pickle
@@ -23,7 +23,7 @@ class AsyncFunction(ForkProcess):
 
def _start(self):
pr, pw = os.pipe()
-   self.fd_pipes = {}
+   self.fd_pipes = {} if self.fd_pipes is None else self.fd_pipes
self.fd_pipes[pw] = pw
self._async_func_reader_pw = pw
self._async_func_reader = PipeReader(

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/, lib/portage/util/, ...

2020-07-17 Thread Michał Górny 
commit: 323df713a741440795c565ee2d79de0beb22b722
Author: Michał Górny  gentoo  org>
AuthorDate: Fri Jul 17 04:30:35 2020 +
Commit: Michał Górny  gentoo  org>
CommitDate: Fri Jul 17 04:39:01 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=323df713

Remove support code for Python < 3.2

Signed-off-by: Michał Górny  gentoo.org>

 lib/portage/tests/process/test_poll.py |  9 +++--
 lib/portage/tests/util/futures/asyncio/test_pipe_closed.py |  7 ++-
 lib/portage/util/configparser.py   | 13 +++--
 3 files changed, 8 insertions(+), 21 deletions(-)

diff --git a/lib/portage/tests/process/test_poll.py 
b/lib/portage/tests/process/test_poll.py
index f505b5049..3ea176c0d 100644
--- a/lib/portage/tests/process/test_poll.py
+++ b/lib/portage/tests/process/test_poll.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2019 Gentoo Authors
+# Copyright 1998-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -34,11 +34,8 @@ class PipeReaderTestCase(TestCase):
 
def test_domain_socket(self):
def make_pipes():
-   if sys.version_info >= (3, 2):
-   read_end, write_end = socket.socketpair()
-   return (read_end.detach(), write_end.detach()), 
None
-   else:
-   self.skipTest('socket detach not supported')
+   read_end, write_end = socket.socketpair()
+   return (read_end.detach(), write_end.detach()), None
self._do_test(make_pipes)
 
def test_named_pipe(self):

diff --git a/lib/portage/tests/util/futures/asyncio/test_pipe_closed.py 
b/lib/portage/tests/util/futures/asyncio/test_pipe_closed.py
index 507385c04..b8e5556d0 100644
--- a/lib/portage/tests/util/futures/asyncio/test_pipe_closed.py
+++ b/lib/portage/tests/util/futures/asyncio/test_pipe_closed.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -32,10 +32,7 @@ class _PipeClosedTestCase(object):
self._do_test(read_end, write_end)
 
def test_domain_socket(self):
-   if sys.version_info >= (3, 2):
-   read_end, write_end = socket.socketpair()
-   else:
-   self.skipTest('socket detach not supported')
+   read_end, write_end = socket.socketpair()
self._do_test(read_end.detach(), write_end.detach())
 
def test_named_pipe(self):

diff --git a/lib/portage/util/configparser.py b/lib/portage/util/configparser.py
index f3452231f..9bd9f9722 100644
--- a/lib/portage/util/configparser.py
+++ b/lib/portage/util/configparser.py
@@ -11,16 +11,9 @@ __all__ = ['ConfigParserError', 'NoOptionError', 
'ParsingError',
 import io
 import sys
 
-try:
-   from configparser import (Error as ConfigParserError,
-   NoOptionError, ParsingError, RawConfigParser)
-   if sys.hexversion >= 0x302:
-   from configparser import ConfigParser as SafeConfigParser
-   else:
-   from configparser import SafeConfigParser
-except ImportError:
-   from ConfigParser import (Error as ConfigParserError,
-   NoOptionError, ParsingError, RawConfigParser, SafeConfigParser)
+from configparser import (Error as ConfigParserError,
+   NoOptionError, ParsingError, RawConfigParser)
+from configparser import ConfigParser as SafeConfigParser
 
 from portage import _encodings
 from portage import _unicode_encode

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2019-08-30 Thread Zac Medico 
commit: a6b9b218bc8599a5c07470aabc41a7af6a1f05d7
Author: Mike Gilbert  gentoo  org>
AuthorDate: Fri Aug 30 18:24:48 2019 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Aug 31 03:10:14 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a6b9b218

Add test case for unshare_net code in portage.process

Code by Zac Medico, with some small tweaks.

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/tests/process/test_unshare_net.py | 38 +++
 1 file changed, 38 insertions(+)

diff --git a/lib/portage/tests/process/test_unshare_net.py 
b/lib/portage/tests/process/test_unshare_net.py
new file mode 100644
index 0..745b79a47
--- /dev/null
+++ b/lib/portage/tests/process/test_unshare_net.py
@@ -0,0 +1,38 @@
+# Copyright 2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import os
+import platform
+
+import portage.process
+from portage.const import BASH_BINARY
+from portage.tests import TestCase
+
+CLONE_NEWNET = 0x4000
+
+UNSHARE_NET_TEST_SCRIPT = """
+ping -c 1 -W 1 127.0.0.1 || exit 1
+ping -c 1 -W 1 10.0.0.1 || exit 1
+[[ -n ${IPV6} ]] || exit 0
+ping -c 1 -W 1 ::1 || exit 1
+ping -c 1 -W 1 fd::1 || exit 1
+"""
+
+class UnshareNetTestCase(TestCase):
+
+   def testUnshareNet(self):
+
+   if platform.system() != 'Linux':
+   self.skipTest('not Linux')
+   if portage.process.find_binary('ping') is None:
+   self.skipTest('ping not found')
+
+   errno_value = portage.process._unshare_validate(CLONE_NEWNET)
+   if errno_value != 0:
+   self.skipTest("Unable to unshare: %s" % (
+   errno.errorcode.get(errno_value, '?')))
+
+   env = os.environ.copy()
+   env['IPV6'] = '1' if portage.process._has_ipv6() else ''
+   self.assertEqual(portage.process.spawn([BASH_BINARY, '-c', 
UNSHARE_NET_TEST_SCRIPT], unshare_net=True, env=env), 0)

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/

2019-04-14 Thread Zac Medico 
commit: ca08349fbec86a323037d5391717c10abc0421a8
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Apr 14 19:42:36 2019 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Apr 14 19:44:21 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ca08349f

PipeReaderTestCase: use asyncio.create_subprocess_exec

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/tests/process/test_poll.py | 20 +---
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/lib/portage/tests/process/test_poll.py 
b/lib/portage/tests/process/test_poll.py
index f700a5585..f505b5049 100644
--- a/lib/portage/tests/process/test_poll.py
+++ b/lib/portage/tests/process/test_poll.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2018 Gentoo Foundation
+# Copyright 1998-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -6,13 +6,12 @@ import pty
 import shutil
 import socket
 import sys
-import subprocess
 import tempfile
 
 from portage import os
 from portage.tests import TestCase
-from portage.util._async.PopenProcess import PopenProcess
 from portage.util._eventloop.global_event_loop import global_event_loop
+from portage.util.futures import asyncio
 from _emerge.PipeReader import PipeReader
 
 class PipeReaderTestCase(TestCase):
@@ -68,17 +67,16 @@ class PipeReaderTestCase(TestCase):
input_files={"producer" : master_file},
_use_array=self._use_array,
scheduler=scheduler)
+   consumer.start()
 
-   producer = PopenProcess(
-   pipe_reader=consumer,
-   proc=subprocess.Popen(["bash", "-c", self._echo_cmd % 
test_string],
-   stdout=slave_fd),
-   scheduler=scheduler)
+   producer = 
scheduler.run_until_complete(asyncio.create_subprocess_exec(
+   "bash", "-c", self._echo_cmd % test_string,
+   stdout=slave_fd,
+   loop=scheduler))
 
-   producer.start()
os.close(slave_fd)
-   producer.wait()
-   consumer.wait()
+   scheduler.run_until_complete(producer.wait())
+   scheduler.run_until_complete(consumer.async_wait())
 
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)