[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2024-02-22 Thread Zac Medico 
commit: d718cea94a180042b2285698b2c19113c5d25987
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Feb 22 06:41:49 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Feb 22 07:28:38 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d718cea9

_get_running_loop: Support real asyncio.run

When called via the real asyncio.run implementation, wrap
the running asyncio loop. Otherwise, it's not possible to
call portage libraries via the real asyncio.run without
triggering Future "attached to a different loop" errors.

Bug: https://bugs.gentoo.org/761538
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/__init__.py | 28 +--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index 22241f335d..4eecc46a89 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -325,13 +325,37 @@ def _safe_loop():
 
 
 def _get_running_loop():
+"""
+This calls the real asyncio get_running_loop() and wraps that with
+portage's internal AsyncioEventLoop wrapper. If there is no running
+asyncio event loop but portage has a reference to another running
+loop in this thread, then use that instead.
+
+This behavior enables portage internals to use the real asyncio.run
+while remaining compatible with internal code that does not use the
+real asyncio.run.
+"""
+try:
+_loop = _real_asyncio.get_running_loop()
+except RuntimeError:
+_loop = None
+
 with _thread_weakrefs.lock:
 if _thread_weakrefs.pid == portage.getpid():
 try:
 loop = _thread_weakrefs.loops[threading.get_ident()]
 except KeyError:
-return None
-return loop if loop.is_running() else None
+pass
+else:
+if _loop is loop._loop:
+return loop
+elif _loop is None:
+return loop if loop.is_running() else None
+
+# If _loop it not None here it means it was probably a temporary
+# loop created by asyncio.run, so we don't try to cache it, and
+# just return a temporary wrapper.
+return None if _loop is None else _AsyncioEventLoop(loop=_loop)
 
 
 def _thread_weakrefs_atexit():

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/package/ebuild/, ...

2024-02-21 Thread Zac Medico 
commit: 18cdb6331a66c1cc92f296b1aaf0538f63586875
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Feb 13 08:44:50 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=18cdb633

EbuildPhase: async_check_locale

Change config.environ() check_locale calls to async_check_locale
calls in the EbuildPhase _async_start method in order to eliminate
synchronous waiting for child processes in the main event loop
thread.

Bug: https://bugs.gentoo.org/923841
Signed-off-by: Zac Medico  gentoo.org>

 lib/_emerge/EbuildMetadataPhase.py|  4 
 lib/_emerge/EbuildPhase.py| 28 ++-
 lib/portage/package/ebuild/config.py  | 26 +++--
 lib/portage/util/futures/_asyncio/__init__.py |  9 +
 lib/portage/util/locale.py| 28 ++-
 5 files changed, 70 insertions(+), 25 deletions(-)

diff --git a/lib/_emerge/EbuildMetadataPhase.py 
b/lib/_emerge/EbuildMetadataPhase.py
index fc64d84c94..54177840c7 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,6 +8,7 @@ import portage
 
 portage.proxy.lazyimport.lazyimport(
 globals(),
+"_emerge.EbuildPhase:_setup_locale",
 "portage.package.ebuild._metadata_invalid:eapi_invalid",
 )
 from portage import os
@@ -83,6 +84,9 @@ class EbuildMetadataPhase(SubProcess):
 settings.setcpv(self.cpv)
 settings.configdict["pkg"]["EAPI"] = parsed_eapi
 
+# This requires above setcpv and EAPI setup.
+await _setup_locale(self.settings)
+
 debug = settings.get("PORTAGE_DEBUG") == "1"
 master_fd = None
 slave_fd = None

diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 3b366f39c7..b472803438 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import (
 _prepare_fake_distdir,
 _prepare_fake_filesdir,
 )
+from portage.eapi import _get_eapi_attrs
 from portage.util import writemsg, ensure_dirs
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
 from portage.util._async.BuildLogger import BuildLogger
@@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport(
 + "_post_src_install_write_metadata,"
 + "_preinst_bsdflags",
 "portage.util.futures.unix_events:_set_nonblocking",
+"portage.util.locale:async_check_locale,split_LC_ALL",
 )
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
 
 
+async def _setup_locale(settings):
+eapi_attrs = _get_eapi_attrs(settings["EAPI"])
+if eapi_attrs.posixish_locale:
+split_LC_ALL(settings)
+settings["LC_COLLATE"] = "C"
+# check_locale() returns None when check can not be executed.
+if await async_check_locale(silent=True, env=settings.environ()) is 
False:
+# try another locale
+for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
+settings["LC_CTYPE"] = l
+if await async_check_locale(silent=True, 
env=settings.environ()):
+# TODO: output the following only once
+# writemsg(
+# _("!!! LC_CTYPE unsupported, using %s instead\n")
+# % self.settings["LC_CTYPE"]
+# )
+break
+else:
+raise AssertionError("C locale did not pass the test!")
+
+
 class EbuildPhase(CompositeTask):
 __slots__ = ("actionmap", "fd_pipes", "phase", "settings") + 
("_ebuild_lock",)
 
@@ -95,6 +118,9 @@ class EbuildPhase(CompositeTask):
 self._start_task(AsyncTaskFuture(future=future), 
self._async_start_exit)
 
 async def _async_start(self):
+
+await _setup_locale(self.settings)
+
 need_builddir = self.phase not in 
EbuildProcess._phases_without_builddir
 
 if need_builddir:

diff --git a/lib/portage/package/ebuild/config.py 
b/lib/portage/package/ebuild/config.py
index c89354cbf7..bafdc55a08 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -29,7 +29,6 @@ portage.proxy.lazyimport.lazyimport(
 "portage.dbapi.vartree:vartree",
 "portage.package.ebuild.doebuild:_phase_func_map",
 "portage.util.compression_probe:_compressors",
-"portage.util.locale:check_locale,split_LC_ALL",
 )
 from portage import bsd_chflags, load_mod, os, selinux, _unicode_decode
 from portage.const import (
@@ -3371,20 +3370,17 @@ class config:
 mydict["EBUILD_PHASE_FUNC"] = phase_func
 
 if eapi_attrs.posixish_locale:
-split_LC_ALL(mydict)
-

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2024-02-21 Thread Zac Medico 
commit: a42c2164ada634262ae1f791ad60298fe3468a94
Author: Zac Medico  gentoo  org>
AuthorDate: Tue Feb 13 03:39:35 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a42c2164

asyncio: Wrap asyncio.Lock for python 3.9 compat

Wrap asyncio.Lock for compatibility with python 3.9 where the
deprecated loop parameter is required in order to avoid "got
Future  attached to a different loop" errors.

The pordbapi async_aux_get method can use asyncio.Lock to
serialize access to its doebuild_settings attribute in order
to prevent issues like bug 924319.

Bug: https://bugs.gentoo.org/924319
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/__init__.py | 17 +
 1 file changed, 17 insertions(+)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index 8f1b8e8275..b6481c281e 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -9,6 +9,7 @@ __all__ = (
 "CancelledError",
 "Future",
 "InvalidStateError",
+"Lock",
 "TimeoutError",
 "get_child_watcher",
 "get_event_loop",
@@ -22,6 +23,7 @@ __all__ = (
 "wait_for",
 )
 
+import sys
 import types
 import weakref
 
@@ -35,6 +37,7 @@ from asyncio import (
 FIRST_EXCEPTION,
 Future,
 InvalidStateError,
+Lock as _Lock,
 shield,
 TimeoutError,
 wait_for,
@@ -159,6 +162,20 @@ def iscoroutinefunction(func):
 return False
 
 
+class Lock(_Lock):
+"""
+Inject loop parameter for python3.9 or less in order to avoid
+"got Future  attached to a different loop" errors.
+"""
+
+def __init__(self, **kwargs):
+if sys.version_info >= (3, 10):
+kwargs.pop("loop", None)
+elif "loop" not in kwargs:
+kwargs["loop"] = _safe_loop()._loop
+super().__init__(**kwargs)
+
+
 class Task(Future):
 """
 Schedule the execution of a coroutine: wrap it in a future. A task

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/util/, ...

2024-02-09 Thread Zac Medico 
commit: c95fc64abf9698263090b3ffd4a056e989dd2be1
Author: Zac Medico  gentoo  org>
AuthorDate: Fri Feb  9 06:38:41 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Fri Feb  9 08:19:00 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c95fc64a

EbuildPhase: async_check_locale

Change config.environ() check_locale calls to async_check_locale
calls in the EbuildPhase _async_start method in order to eliminate
synchronous waiting for child processes in the main event loop
thread.

Bug: https://bugs.gentoo.org/923841
Signed-off-by: Zac Medico  gentoo.org>

 lib/_emerge/EbuildMetadataPhase.py| 21 
 lib/_emerge/EbuildPhase.py| 28 ++-
 lib/portage/package/ebuild/config.py  | 26 +++--
 lib/portage/util/futures/_asyncio/__init__.py |  9 +
 lib/portage/util/locale.py| 28 ++-
 5 files changed, 87 insertions(+), 25 deletions(-)

diff --git a/lib/_emerge/EbuildMetadataPhase.py 
b/lib/_emerge/EbuildMetadataPhase.py
index f4f685e81c..53b7ad9624 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,12 +8,14 @@ import portage
 
 portage.proxy.lazyimport.lazyimport(
 globals(),
+"_emerge.EbuildPhase:_setup_locale",
 "portage.package.ebuild._metadata_invalid:eapi_invalid",
 )
 from portage import os
 from portage import _encodings
 from portage import _unicode_decode
 from portage import _unicode_encode
+from portage.util.futures import asyncio
 
 import fcntl
 
@@ -44,6 +46,12 @@ class EbuildMetadataPhase(SubProcess):
 _files_dict = slot_dict_class(_file_names, prefix="")
 
 def _start(self):
+asyncio.ensure_future(
+self._async_start(), loop=self.scheduler
+).add_done_callback(self._async_start_done)
+
+async def _async_start(self):
+
 ebuild_path = self.ebuild_hash.location
 
 with open(
@@ -75,6 +83,9 @@ class EbuildMetadataPhase(SubProcess):
 settings.setcpv(self.cpv)
 settings.configdict["pkg"]["EAPI"] = parsed_eapi
 
+# This requires above setcpv and EAPI setup.
+await _setup_locale(self.settings)
+
 debug = settings.get("PORTAGE_DEBUG") == "1"
 master_fd = None
 slave_fd = None
@@ -139,6 +150,16 @@ class EbuildMetadataPhase(SubProcess):
 
 self._proc = retval
 
+def _async_start_done(self, future):
+future.cancelled() or future.result()
+if future.cancelled():
+self.cancel()
+self._was_cancelled()
+
+if self.returncode is not None:
+self._unregister()
+self.wait()
+
 def _output_handler(self):
 while True:
 buf = self._read_buf(self._files.ebuild)

diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index c81bf54a81..c8caf73722 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import (
 _prepare_fake_distdir,
 _prepare_fake_filesdir,
 )
+from portage.eapi import _get_eapi_attrs
 from portage.util import writemsg, ensure_dirs
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
 from portage.util._async.BuildLogger import BuildLogger
@@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport(
 + "_post_src_install_write_metadata,"
 + "_preinst_bsdflags",
 "portage.util.futures.unix_events:_set_nonblocking",
+"portage.util.locale:async_check_locale,split_LC_ALL",
 )
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
 
 
+async def _setup_locale(settings):
+eapi_attrs = _get_eapi_attrs(settings["EAPI"])
+if eapi_attrs.posixish_locale:
+split_LC_ALL(settings)
+settings["LC_COLLATE"] = "C"
+# check_locale() returns None when check can not be executed.
+if await async_check_locale(silent=True, env=settings.environ()) is 
False:
+# try another locale
+for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
+settings["LC_CTYPE"] = l
+if await async_check_locale(silent=True, 
env=settings.environ()):
+# TODO: output the following only once
+# writemsg(
+# _("!!! LC_CTYPE unsupported, using %s instead\n")
+# % self.settings["LC_CTYPE"]
+# )
+break
+else:
+raise AssertionError("C locale did not pass the test!")
+
+
 class EbuildPhase(CompositeTask):
 __slots__ = ("actionmap", "fd_pipes", "phase", "settings") + 
("_ebuild_lock",)
 
@@ -94,6 +117,9 @@ class

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/sync/modules/rsync/

2024-02-06 Thread Zac Medico 
commit: 62332ee82b8b88fa5a65aafa7c221ccdaa7d65a8
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Feb  4 00:11:07 2024 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Feb  7 00:55:46 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=62332ee8

RsyncSync: Migrate to spawn returnproc parameter

Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/sync/modules/rsync/rsync.py   | 40 +--
 lib/portage/util/futures/_asyncio/__init__.py |  6 +++-
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/lib/portage/sync/modules/rsync/rsync.py 
b/lib/portage/sync/modules/rsync/rsync.py
index 175c7f2e8e..5d442d2626 100644
--- a/lib/portage/sync/modules/rsync/rsync.py
+++ b/lib/portage/sync/modules/rsync/rsync.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import datetime
@@ -708,48 +708,47 @@ class RsyncSync(NewBase):
 command.append(syncuri.rstrip("/") + "/metadata/timestamp.chk")
 command.append(tmpservertimestampfile)
 content = None
-pids = []
+proc = None
+proc_waiter = None
+loop = asyncio.get_event_loop()
 try:
 # Timeout here in case the server is unresponsive.  The
 # --timeout rsync option doesn't apply to the initial
 # connection attempt.
 try:
-if self.rsync_initial_timeout:
-
portage.exception.AlarmSignal.register(self.rsync_initial_timeout)
-
-pids.extend(
-portage.process.spawn(command, returnpid=True, 
**self.spawn_kwargs)
+proc = portage.process.spawn(
+command, returnproc=True, **self.spawn_kwargs
+)
+proc_waiter = asyncio.ensure_future(proc.wait(), loop)
+future = (
+asyncio.wait_for(
+asyncio.shield(proc_waiter), self.rsync_initial_timeout
+)
+if self.rsync_initial_timeout
+else proc_waiter
 )
-exitcode = os.waitpid(pids[0], 0)[1]
+exitcode = loop.run_until_complete(future)
 if self.usersync_uid is not None:
 portage.util.apply_permissions(
 tmpservertimestampfile, uid=os.getuid()
 )
 content = portage.grabfile(tmpservertimestampfile)
 finally:
-if self.rsync_initial_timeout:
-portage.exception.AlarmSignal.unregister()
 try:
 os.unlink(tmpservertimestampfile)
 except OSError:
 pass
-except portage.exception.AlarmSignal:
+except (TimeoutError, asyncio.TimeoutError):
 # timed out
 print("timed out")
 # With waitpid and WNOHANG, only check the
 # first element of the tuple since the second
 # element may vary (bug #337465).
-if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
-os.kill(pids[0], signal.SIGTERM)
-os.waitpid(pids[0], 0)
+if proc_waiter and not proc_waiter.done():
+proc.terminate()
+loop.run_until_complete(proc_waiter)
 # This is the same code rsync uses for timeout.
 exitcode = 30
-else:
-if exitcode != os.EX_OK:
-if exitcode & 0xFF:
-exitcode = (exitcode & 0xFF) << 8
-else:
-exitcode = exitcode >> 8
 
 if content:
 try:
@@ -758,7 +757,6 @@ class RsyncSync(NewBase):
 )
 except (OverflowError, ValueError):
 pass
-del command, pids, content
 
 if exitcode == os.EX_OK:
 if (servertimestamp != 0) and (servertimestamp == timestamp):

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index a5a6cb3a5b..8f1b8e8275 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018-2021 Gentoo Authors
+# Copyright 2018-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = (
@@ -15,9 +15,11 @@ __all__ = (
 "set_child_watcher",
 "get_event_loop_policy",
 "set_event_loop_policy",
+"shield",
 "sleep",
 "Task",
 "wait",
+"wait_for",
 )
 
 import types
@@ -33,7 +35,9 @@ from asyncio import (
 FIRST_EXCEPTION,
 Future,
 InvalidStateError,
+shield,
 TimeoutError,
+wait_for,
 )
 
 import threading

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/tests/process/, lib/_emerge/, ...

2023-06-13 Thread Sam James 
commit: efbc9c8d38b742a37968fc3b33d3dffdaf3be9f2
Author: Berin Aniesh  gmail  com>
AuthorDate: Fri Jun  9 02:00:45 2023 +
Commit: Sam James  gentoo  org>
CommitDate: Wed Jun 14 01:44:02 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=efbc9c8d

Remove dummy_threading

Portage's MSV is python 3.7 and dummy threading is deprecated
since python 3.7. So everything dummy_threading is removed.

Signed-off-by: Berin Aniesh  gmail.com>
Closes: https://github.com/gentoo/portage/pull/1052
Signed-off-by: Sam James  gentoo.org>

 lib/_emerge/AsynchronousLock.py| 29 
 lib/_emerge/PollScheduler.py   |  7 ++---
 lib/portage/_emirrordist/MirrorDistTask.py |  6 +
 lib/portage/debug.py   |  8 ++
 lib/portage/locks.py   |  7 +++--
 lib/portage/proxy/lazyimport.py|  8 ++
 lib/portage/tests/locks/test_asynchronous_lock.py  | 31 --
 .../tests/process/test_PopenProcessBlockingIO.py   | 14 +-
 lib/portage/tests/util/futures/test_retry.py   |  7 ++---
 lib/portage/util/_async/PipeReaderBlockingIO.py|  8 ++
 lib/portage/util/futures/_asyncio/__init__.py  |  5 +---
 11 files changed, 34 insertions(+), 96 deletions(-)

diff --git a/lib/_emerge/AsynchronousLock.py b/lib/_emerge/AsynchronousLock.py
index 1a69d0847..c9c8e7f3c 100644
--- a/lib/_emerge/AsynchronousLock.py
+++ b/lib/_emerge/AsynchronousLock.py
@@ -1,19 +1,10 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import fcntl
 import logging
 import sys
-
-try:
-import dummy_threading
-except ImportError:
-dummy_threading = None
-
-try:
-import threading
-except ImportError:
-threading = dummy_threading
+import threading
 
 import portage
 from portage import os
@@ -39,7 +30,6 @@ class AsynchronousLock(AsynchronousTask):
 __slots__ = ("path",) + (
 "_imp",
 "_force_async",
-"_force_dummy",
 "_force_process",
 "_force_thread",
 "_unlock_future",
@@ -61,14 +51,11 @@ class AsynchronousLock(AsynchronousTask):
 return
 
 if self._force_process or (
-not self._force_thread
-and (self._use_process_by_default or threading is dummy_threading)
+not self._force_thread and self._use_process_by_default
 ):
 self._imp = _LockProcess(path=self.path, scheduler=self.scheduler)
 else:
-self._imp = _LockThread(
-path=self.path, scheduler=self.scheduler, 
_force_dummy=self._force_dummy
-)
+self._imp = _LockThread(path=self.path, scheduler=self.scheduler)
 
 self._imp.addExitListener(self._imp_exit)
 self._imp.start()
@@ -115,19 +102,13 @@ class _LockThread(AbstractPollTask):
 using a background thread. After the lock is acquired, the thread
 writes to a pipe in order to notify a poll loop running in the main
 thread.
-
-If the threading module is unavailable then the dummy_threading
-module will be used, and the lock will be acquired synchronously
-(before the start() method returns).
 """
 
-__slots__ = ("path",) + ("_force_dummy", "_lock_obj", "_thread", 
"_unlock_future")
+__slots__ = ("path",) + ("_lock_obj", "_thread", "_unlock_future")
 
 def _start(self):
 self._registered = True
 threading_mod = threading
-if self._force_dummy:
-threading_mod = dummy_threading
 self._thread = threading_mod.Thread(target=self._run_lock)
 self._thread.daemon = True
 self._thread.start()

diff --git a/lib/_emerge/PollScheduler.py b/lib/_emerge/PollScheduler.py
index 73f2affde..e5dffd8af 100644
--- a/lib/_emerge/PollScheduler.py
+++ b/lib/_emerge/PollScheduler.py
@@ -1,10 +1,7 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-try:
-import threading
-except ImportError:
-import dummy_threading as threading
+import threading
 
 from portage.util.futures import asyncio
 from portage.util._async.SchedulerInterface import SchedulerInterface

diff --git a/lib/portage/_emirrordist/MirrorDistTask.py 
b/lib/portage/_emirrordist/MirrorDistTask.py
index 5b2543c13..2925394a8 100644
--- a/lib/portage/_emirrordist/MirrorDistTask.py
+++ b/lib/portage/_emirrordist/MirrorDistTask.py
@@ -4,11 +4,7 @@
 import errno
 import logging
 import time
-
-try:
-import threading
-except ImportError:
-import dummy_threading as threading
+import threading
 
 import portage
 from portage import os

diff --git a/lib/portage/debug.py b/lib/portage/debug.py
index b7106b799..ee2dc13e7 100644
--- a/lib/portage/debug.py
+++ b/lib/portage/debug.py
@@ -1,13 +1,9 @@
-# Copyright

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2021-09-19 Thread Zac Medico 
commit: 631bedffe29124d693de3b539fc908d9feec1420
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Sep 20 05:05:38 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Sep 20 05:08:47 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=631bedff

_safe_loop: fix python3.10 DeprecationWarning

DeprecationWarning: There is no current event loop

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/__init__.py | 10 +++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index c1229528a..ccf800c66 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -280,10 +280,14 @@ def _safe_loop():
 loop = _thread_weakrefs.loops[thread_key]
 except KeyError:
 try:
-_real_asyncio.get_event_loop()
+try:
+_loop = _real_asyncio.get_running_loop()
+except AttributeError:
+_loop = _real_asyncio.get_event_loop()
 except RuntimeError:
-_real_asyncio.set_event_loop(_real_asyncio.new_event_loop())
-loop = _thread_weakrefs.loops[thread_key] = _AsyncioEventLoop()
+_loop = _real_asyncio.new_event_loop()
+_real_asyncio.set_event_loop(_loop)
+loop = _thread_weakrefs.loops[thread_key] = 
_AsyncioEventLoop(loop=_loop)
 
 if (
 _thread_weakrefs.mainloop is None

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2021-03-07 Thread Zac Medico 
commit: 9eea2af22c9e51475a4adba57fdded3a2a88c886
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Mar  7 14:18:44 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Mar  7 14:37:46 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9eea2af2

_writer: Use async and await syntax

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/streams.py | 13 +
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/streams.py 
b/lib/portage/util/futures/_asyncio/streams.py
index ea5882dd3..7a8d4a3e0 100644
--- a/lib/portage/util/futures/_asyncio/streams.py
+++ b/lib/portage/util/futures/_asyncio/streams.py
@@ -1,4 +1,4 @@
-# Copyright 2018-2020 Gentoo Authors
+# Copyright 2018-2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -9,7 +9,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
'_emerge.PipeReader:PipeReader',
'portage.util.futures:asyncio',
 )
-from portage.util.futures.compat_coroutine import coroutine
 
 
 def _reader(input_file, loop=None):
@@ -55,8 +54,7 @@ class _Reader:
self._pipe_reader = None
 
 
-@coroutine
-def _writer(output_file, content, loop=None):
+async def _writer(output_file, content, loop=DeprecationWarning):
"""
Asynchronously write bytes to output file. The output file is
assumed to be in non-blocking mode. If an EnvironmentError
@@ -68,10 +66,9 @@ def _writer(output_file, content, loop=None):
@type output_file: file object
@param content: content to write
@type content: bytes
-   @param loop: asyncio.AbstractEventLoop (or compatible)
-   @type loop: event loop
+   @param loop: deprecated
"""
-   loop = asyncio._wrap_loop(loop)
+   loop = asyncio.get_event_loop()
fd = output_file.fileno()
while content:
try:
@@ -82,7 +79,7 @@ def _writer(output_file, content, loop=None):
waiter = loop.create_future()
loop.add_writer(fd, lambda: waiter.done() or 
waiter.set_result(None))
try:
-   yield waiter
+   await waiter
finally:
# The loop and output file may have been closed.
if not loop.is_closed():

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2021-03-06 Thread Zac Medico 
commit: 15049a041909f85b02a52f5b1938c6dd4171c9e3
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Mar  7 03:03:40 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Mar  7 05:12:48 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=15049a04

Removed unused portage.util.futures._asyncio.tasks

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/__init__.py | 46 ++---
 lib/portage/util/futures/_asyncio/tasks.py| 96 ---
 2 files changed, 19 insertions(+), 123 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index 207e7205d..4643697e0 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -25,7 +25,16 @@ import types
 import weakref
 
 import asyncio as _real_asyncio
-from asyncio.subprocess import Process
+# pylint: disable=redefined-builtin
+from asyncio import (
+   ALL_COMPLETED,
+   CancelledError,
+   FIRST_COMPLETED,
+   FIRST_EXCEPTION,
+   Future,
+   InvalidStateError,
+   TimeoutError,
+)
 
 try:
import threading
@@ -38,20 +47,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures:compat_coroutine@_compat_coroutine',
 )
 from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as 
_AsyncioEventLoop
-# pylint: disable=redefined-builtin
-from portage.util.futures.futures import (
-   CancelledError,
-   Future,
-   InvalidStateError,
-   TimeoutError,
-)
-# pylint: enable=redefined-builtin
-from portage.util.futures._asyncio.tasks import (
-   ALL_COMPLETED,
-   FIRST_COMPLETED,
-   FIRST_EXCEPTION,
-   wait,
-)
 
 
 _lock = threading.Lock()
@@ -131,20 +126,17 @@ def create_subprocess_exec(*args, **kwargs):
# Python 3.4 and later implement PEP 446, which makes newly
# created file descriptors non-inheritable by default.
kwargs.setdefault('close_fds', False)
-   if isinstance(loop._asyncio_wrapper, _AsyncioEventLoop):
-   # Use the real asyncio create_subprocess_exec (loop argument
-   # is deprecated since since Python 3.8).
-   return _real_asyncio.create_subprocess_exec(*args, **kwargs)
-
-   result = loop.create_future()
+   # Use the real asyncio create_subprocess_exec (loop argument
+   # is deprecated since since Python 3.8).
+   return ensure_future(_real_asyncio.create_subprocess_exec(*args, 
**kwargs), loop=loop)
 
-   result.set_result(Process(subprocess.Popen(
-   args,
-   stdin=kwargs.pop('stdin', None),
-   stdout=kwargs.pop('stdout', None),
-   stderr=kwargs.pop('stderr', None), **kwargs), loop))
 
-   return result
+def wait(futures, loop=None, timeout=None, return_when=ALL_COMPLETED):
+   """
+   Wraps asyncio.wait() and omits the loop argument which is not
+   supported since python 3.10.
+   """
+   return _real_asyncio.wait(futures, timeout=timeout, 
return_when=return_when)
 
 
 def iscoroutinefunction(func):

diff --git a/lib/portage/util/futures/_asyncio/tasks.py 
b/lib/portage/util/futures/_asyncio/tasks.py
deleted file mode 100644
index c9db3146e..0
--- a/lib/portage/util/futures/_asyncio/tasks.py
+++ /dev/null
@@ -1,96 +0,0 @@
-# Copyright 2018-2020 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-___all___ = (
-   'ALL_COMPLETED',
-   'FIRST_COMPLETED',
-   'FIRST_EXCEPTION',
-   'wait',
-)
-
-from asyncio import ALL_COMPLETED, FIRST_COMPLETED, FIRST_EXCEPTION
-
-import portage
-portage.proxy.lazyimport.lazyimport(globals(),
-   'portage.util.futures:asyncio',
-)
-
-def wait(futures, loop=None, timeout=None, return_when=ALL_COMPLETED):
-   """
-   Use portage's internal EventLoop to emulate asyncio.wait:
-   https://docs.python.org/3/library/asyncio-task.html#asyncio.wait
-
-   @param futures: futures to wait for
-   @type futures: asyncio.Future (or compatible)
-   @param timeout: number of seconds to wait (wait indefinitely if
-   not specified)
-   @type timeout: int or float
-   @param return_when: indicates when this function should return, must
-   be one of the constants ALL_COMPLETED, FIRST_COMPLETED, or
-   FIRST_EXCEPTION (default is ALL_COMPLETED)
-   @type return_when: object
-   @param loop: event loop
-   @type loop: EventLoop
-   @return: tuple of (done, pending).
-   @rtype: asyncio.Future (or compatible)
-   """
-   loop = asyncio._wrap_loop(loop)
-   result_future = loop.create_future()
-   _Waiter(futures, timeout, return_when, result_future, loop)
-   return result_future
-
-
-class _Waiter:
-   def __init__(self, futures, timeout, return_when, result_future, loop):
-

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2021-03-06 Thread Zac Medico 
commit: 1e843f853a9afe82d599e6ab09064147ddc1d271
Author: Matt Turner  gentoo  org>
AuthorDate: Thu Mar  4 19:24:58 2021 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Mar  6 09:06:53 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1e843f85

Use asyncio.subprocess.Process directly

With no need to support Python 2, we can remove our private
implementation.

Signed-off-by: Matt Turner  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/__init__.py |   8 +-
 lib/portage/util/futures/_asyncio/process.py  | 116 --
 2 files changed, 4 insertions(+), 120 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index 5590963f1..207e7205d 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -25,6 +25,7 @@ import types
 import weakref
 
 import asyncio as _real_asyncio
+from asyncio.subprocess import Process
 
 try:
import threading
@@ -45,7 +46,6 @@ from portage.util.futures.futures import (
TimeoutError,
 )
 # pylint: enable=redefined-builtin
-from portage.util.futures._asyncio.process import _Process
 from portage.util.futures._asyncio.tasks import (
ALL_COMPLETED,
FIRST_COMPLETED,
@@ -124,8 +124,8 @@ def create_subprocess_exec(*args, **kwargs):
@type loop: event loop
@type kwargs: varies
@param kwargs: subprocess.Popen parameters
-   @rtype: asyncio.Future (or compatible)
-   @return: subset of asyncio.subprocess.Process interface
+   @rtype: asyncio.subprocess.Process (or compatible)
+   @return: asyncio.subprocess.Process interface
"""
loop = _wrap_loop(kwargs.pop('loop', None))
# Python 3.4 and later implement PEP 446, which makes newly
@@ -138,7 +138,7 @@ def create_subprocess_exec(*args, **kwargs):
 
result = loop.create_future()
 
-   result.set_result(_Process(subprocess.Popen(
+   result.set_result(Process(subprocess.Popen(
args,
stdin=kwargs.pop('stdin', None),
stdout=kwargs.pop('stdout', None),

diff --git a/lib/portage/util/futures/_asyncio/process.py 
b/lib/portage/util/futures/_asyncio/process.py
deleted file mode 100644
index 275c9031a..0
--- a/lib/portage/util/futures/_asyncio/process.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# Copyright 2018-2020 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-import os
-
-import portage
-portage.proxy.lazyimport.lazyimport(globals(),
-   'portage.util.futures:asyncio',
-   'portage.util.futures.unix_events:_set_nonblocking',
-)
-from portage.util.futures._asyncio.streams import _reader, _writer
-from portage.util.futures.compat_coroutine import coroutine, coroutine_return
-
-
-class _Process:
-   """
-   Emulate a subset of the asyncio.subprocess.Process interface,
-   for python2.
-   """
-   def __init__(self, proc, loop):
-   """
-   @param proc: process instance
-   @type proc: subprocess.Popen
-   @param loop: asyncio.AbstractEventLoop (or compatible)
-   @type loop: event loop
-   """
-   self._proc = proc
-   self._loop = loop
-   self.terminate = proc.terminate
-   self.kill = proc.kill
-   self.send_signal = proc.send_signal
-   self.pid = proc.pid
-   self._waiters = []
-   loop._asyncio_child_watcher.\
-   add_child_handler(self.pid, self._proc_exit)
-
-   @property
-   def returncode(self):
-   return self._proc.returncode
-
-   @coroutine
-   def communicate(self, input=None, loop=None): # pylint: 
disable=redefined-builtin
-   """
-   Read data from stdout and stderr, until end-of-file is reached.
-   Wait for process to terminate.
-
-   @param input: stdin content to write
-   @type input: bytes
-   @return: tuple (stdout_data, stderr_data)
-   @rtype: asyncio.Future (or compatible)
-   """
-   loop = asyncio._wrap_loop(loop or self._loop)
-   futures = []
-   for input_file in (self._proc.stdout, self._proc.stderr):
-   if input_file is None:
-   future = loop.create_future()
-   future.set_result(None)
-   else:
-   future = _reader(input_file, loop=loop)
-   futures.append(future)
-
-   writer = None
-   if input is not None:
-   if self._proc.stdin is None:
-   raise TypeError('communicate: expected file or 
int, got

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2020-12-07 Thread Zac Medico 
commit: dcbcac809213537afaa6b4f9822146a2e984f773
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Dec  7 06:05:04 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Dec  7 07:48:10 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dcbcac80

_safe_loop: instantiate asyncio loop for API consumer thread

In order to maintain compatibility with an API consumer thread which
has not instantiated an asyncio loop for the current thread prior
to calling the portage API, instantiate a loop on its behalf. Since
a ResourceWarning will be triggered if the loop has not been closed
before the process exits, add the loop to a WeakValueDictionary,
and close it if it still exists during exit for the current pid.

Fixes: cecd2f8a259c ("Use default asyncio event loop implementation in API 
consumer threads")
Bug: https://bugs.gentoo.org/758755
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/__init__.py | 49 ++-
 1 file changed, 40 insertions(+), 9 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index 6f3395a91..d39f31786 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -21,7 +21,8 @@ __all__ = (
 )
 
 import subprocess
-import sys
+import types
+import weakref
 
 import asyncio as _real_asyncio
 
@@ -249,23 +250,53 @@ def _safe_loop():
the main thread, this returns a globally shared event loop instance.
 
For external API consumers calling from a non-main thread, an
-   asyncio loop must be registered for the current thread, or else an
-   error will be raised like this:
+   asyncio loop must be registered for the current thread, or else the
+   asyncio.get_event_loop() function will raise an error like this:
 
  RuntimeError: There is no current event loop in thread 'Thread-1'.
 
-   In order to avoid this RuntimeError, the external API consumer
-   is responsible for setting an event loop and managing its lifecycle.
-   For example, this code will set an event loop for the current thread:
+   In order to avoid this RuntimeError, a loop will be automatically
+   created like this:
 
  asyncio.set_event_loop(asyncio.new_event_loop())
 
-   In order to avoid a ResourceWarning, the caller should also close the
-   corresponding loop before the current thread terminates.
+   In order to avoid a ResourceWarning, automatically created loops
+   are added to a WeakValueDictionary, and closed via an atexit hook
+   if they still exist during exit for the current pid.
 
@rtype: asyncio.AbstractEventLoop (or compatible)
@return: event loop instance
"""
if portage._internal_caller or threading.current_thread() is 
threading.main_thread():
return _global_event_loop()
-   return _AsyncioEventLoop()
+
+   thread_key = threading.get_ident()
+   with _thread_weakrefs.lock:
+   if _thread_weakrefs.pid != portage.getpid():
+   _thread_weakrefs.pid = portage.getpid()
+   _thread_weakrefs.loops = weakref.WeakValueDictionary()
+   try:
+   loop = _thread_weakrefs.loops[thread_key]
+   except KeyError:
+   try:
+   _real_asyncio.get_event_loop()
+   except RuntimeError:
+   
_real_asyncio.set_event_loop(_real_asyncio.new_event_loop())
+   loop = _thread_weakrefs.loops[thread_key] = 
_AsyncioEventLoop()
+   return loop
+
+
+def _thread_weakrefs_atexit():
+   with _thread_weakrefs.lock:
+   if _thread_weakrefs.pid == portage.getpid():
+   while True:
+   try:
+   thread_key, loop = 
_thread_weakrefs.loops.popitem()
+   except KeyError:
+   break
+   else:
+   loop.close()
+
+
+_thread_weakrefs = types.SimpleNamespace(lock=threading.Lock(), loops=None, 
pid=None)
+portage.process.atexit_register(_thread_weakrefs_atexit)

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2020-08-03 Thread Zac Medico 
commit: d4979a4a2fc6e99a48bd417eae26ce77ae288444
Author: Aaron Bauman  gentoo  org>
AuthorDate: Mon Aug  3 19:06:04 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Aug  3 19:22:48 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d4979a4a

lib/portage/util/futures/_asyncio/tasks.py: drop unused-import

* Drop unused import
* Update copyright

Signed-off-by: Aaron Bauman  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/tasks.py | 6 +-
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/tasks.py 
b/lib/portage/util/futures/_asyncio/tasks.py
index 84c6f4462..c9db3146e 100644
--- a/lib/portage/util/futures/_asyncio/tasks.py
+++ b/lib/portage/util/futures/_asyncio/tasks.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 ___all___ = (
@@ -14,10 +14,6 @@ import portage
 portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures:asyncio',
 )
-from portage.util._eventloop.global_event_loop import (
-   global_event_loop as _global_event_loop,
-)
-
 
 def wait(futures, loop=None, timeout=None, return_when=ALL_COMPLETED):
"""

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/_sets/, ...

2020-07-23 Thread Zac Medico 
commit: 550727af87d5f646617e0c19a3f3300c8117e7f5
Author: Alec Warner  gentoo  org>
AuthorDate: Fri Jul 24 01:06:54 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Fri Jul 24 02:10:43 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=550727af

Fix redefined-builtin errors for pylint.

Some of these are simple variable renames. A few of the uses were
refactored and some were simply disabled; often due to redefinition as
part of a function signature. I did not do any research in terms of API
changes so these were typically left as-is.

Closes: https://github.com/gentoo/portage/pull/587
Signed-off-by: Alec Warner  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/_emerge/actions.py|  2 +-
 lib/_emerge/depgraph.py   |  2 +-
 lib/_emerge/help.py   |  2 +-
 lib/_emerge/main.py   |  2 +-
 lib/_emerge/resolver/output_helpers.py|  2 +-
 lib/_emerge/resolver/slot_collision.py| 28 +++
 lib/_emerge/search.py |  2 +-
 lib/portage/_sets/dbapi.py|  8 +++
 lib/portage/cache/ebuild_xattr.py | 32 +--
 lib/portage/cache/fs_template.py  |  4 ++--
 lib/portage/cvstree.py| 10 -
 lib/portage/dbapi/porttree.py |  2 +-
 lib/portage/glsa.py   |  4 ++--
 lib/portage/manifest.py   |  4 ++--
 lib/portage/repository/config.py  |  8 +++
 lib/portage/tests/bin/setup_env.py| 18 +++
 lib/portage/tests/sets/shell/testShell.py |  6 ++---
 lib/portage/util/_urlopen.py  |  3 +--
 lib/portage/util/futures/_asyncio/__init__.py |  2 ++
 lib/portage/util/futures/_asyncio/process.py  |  2 +-
 lib/portage/util/futures/events.py|  2 +-
 lib/portage/util/futures/futures.py   |  2 ++
 lib/portage/util/lafilefixer.py   |  8 +++
 23 files changed, 79 insertions(+), 76 deletions(-)

diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 964dca31c..2cbca99d8 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -27,7 +27,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.locale:check_locale',
'portage.emaint.modules.sync.sync:SyncRepos',
'_emerge.chk_updated_cfg_files:chk_updated_cfg_files',
-   '_emerge.help:help@emerge_help',
+   '_emerge.help:emerge_help',
'_emerge.post_emerge:display_news_notification,post_emerge',
'_emerge.stdout_spinner:stdout_spinner',
 )

diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 653348d34..3ff90190d 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -7476,7 +7476,7 @@ class depgraph:
 
mygraph.order.sort(key=cmp_sort_key(cmp_merge_preference))
 
-   def altlist(self, reversed=DeprecationWarning):
+   def altlist(self, reversed=DeprecationWarning): # pylint: 
disable=redefined-builtin
 
if reversed is not DeprecationWarning:
warnings.warn("The reversed parameter of "

diff --git a/lib/_emerge/help.py b/lib/_emerge/help.py
index 2ccd323aa..de3d7b593 100644
--- a/lib/_emerge/help.py
+++ b/lib/_emerge/help.py
@@ -5,7 +5,7 @@ from __future__ import print_function
 
 from portage.output import bold, turquoise, green
 
-def help():
+def emerge_help():
print(bold("emerge:")+" command-line interface to the Portage system")
print(bold("Usage:"))
print("   "+turquoise("emerge")+" [ "+green("options")+" ] [ 
"+green("action")+" ] [ "+turquoise("ebuild")+" | "+turquoise("tbz2")+" | 
"+turquoise("file")+" | "+turquoise("@set")+" | "+turquoise("atom")+" ] [ ... 
]")

diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 69d93f846..8228e0b41 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -16,7 +16,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'textwrap',
'_emerge.actions:load_emerge_config,run_action,' + \
'validate_ebuild_environment',
-   '_emerge.help:help@emerge_help',
+   '_emerge.help:emerge_help',
'_emerge.is_valid_package_atom:insert_category_into_atom'
 )
 from portage import os

diff --git a/lib/_emerge/resolver/output_helpers.py 
b/lib/_emerge/resolver/output_helpers.py
index d5cc9dbcb..25aa925b4 100644
--- a/lib/_emerge/resolver/output_helpers.py
+++ b/lib/_emerge/resolver/output_helpers.py
@@ -469,7 +469,7 @@ def _prune_tree_display(display_list):
del display_list[i]
 
 
-def _calc_changelog(ebuildpath,current,next):
+def _calc_changelog(ebuildpath,current,next): # pylint: 
disable=redefined-builtin
if ebuildpath == None or not os.path.exists(ebuildpath):
return []
current = '-'.join(catpkgsplit(current)[1:])

diff --git

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2020-06-18 Thread Zac Medico 
commit: 92be5a02e452eb0810d2974bc7fa5ee2056ef8e7
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Jun 18 05:33:26 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Jun 18 17:54:37 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=92be5a02

_writer: fix unsafe finally clause (bug 728580)

In the coroutine finally clause, do not call remove_writer in cases
where fd has been closed and then re-allocated to a concurrent
coroutine as in bug 716636.

Also, assume that the caller will put the file in non-blocking mode
and close the file when done, so that this function is suitable for
use within a loop.

Bug: https://bugs.gentoo.org/728580
Reviewed-by: Brian Dolbec  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/process.py | 11 --
 lib/portage/util/futures/_asyncio/streams.py | 50 +---
 2 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/process.py 
b/lib/portage/util/futures/_asyncio/process.py
index 020164c9b..2d3e9b0fd 100644
--- a/lib/portage/util/futures/_asyncio/process.py
+++ b/lib/portage/util/futures/_asyncio/process.py
@@ -1,9 +1,12 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+import os
+
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures:asyncio',
+   'portage.util.futures.unix_events:_set_nonblocking',
 )
 from portage.util.futures._asyncio.streams import _reader, _writer
 from portage.util.futures.compat_coroutine import coroutine, coroutine_return
@@ -59,7 +62,11 @@ class _Process(object):
if input is not None:
if self._proc.stdin is None:
raise TypeError('communicate: expected file or 
int, got {}'.format(type(self._proc.stdin)))
-   writer = 
asyncio.ensure_future(_writer(self._proc.stdin, input), loop=self._loop)
+   stdin = self._proc.stdin
+   stdin = os.fdopen(stdin, 'wb', 0) if isinstance(stdin, 
int) else stdin
+   _set_nonblocking(stdin.fileno())
+   writer = asyncio.ensure_future(_writer(stdin, input, 
loop=self._loop), loop=self._loop)
+   writer.add_done_callback(lambda writer: stdin.close())
 
try:
yield asyncio.wait(futures + [self.wait()], 
loop=self._loop)

diff --git a/lib/portage/util/futures/_asyncio/streams.py 
b/lib/portage/util/futures/_asyncio/streams.py
index 650a16491..870307e1e 100644
--- a/lib/portage/util/futures/_asyncio/streams.py
+++ b/lib/portage/util/futures/_asyncio/streams.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -8,7 +8,6 @@ import portage
 portage.proxy.lazyimport.lazyimport(globals(),
'_emerge.PipeReader:PipeReader',
'portage.util.futures:asyncio',
-   'portage.util.futures.unix_events:_set_nonblocking',
 )
 from portage.util.futures.compat_coroutine import coroutine
 
@@ -59,38 +58,37 @@ class _Reader(object):
 @coroutine
 def _writer(output_file, content, loop=None):
"""
-   Asynchronously write bytes to output file, and close it when
-   done. If an EnvironmentError other than EAGAIN is encountered,
-   which typically indicates that the other end of the pipe has
-   close, the error is raised. This function is a coroutine.
+   Asynchronously write bytes to output file. The output file is
+   assumed to be in non-blocking mode. If an EnvironmentError
+   other than EAGAIN is encountered, which typically indicates that
+   the other end of the pipe has closed, the error is raised.
+   This function is a coroutine.
 
-   @param output_file: output file descriptor
-   @type output_file: file or int
+   @param output_file: output file
+   @type output_file: file object
@param content: content to write
@type content: bytes
@param loop: asyncio.AbstractEventLoop (or compatible)
@type loop: event loop
"""
-   fd = output_file if isinstance(output_file, int) else 
output_file.fileno()
-   _set_nonblocking(fd)
loop = asyncio._wrap_loop(loop)
-   try:
-   while content:
+   fd = output_file.fileno()
+   while content:
+   try:
+   content = content[os.write(fd, content):]
+   except EnvironmentError as e:
+   if e.errno != errno.EAGAIN:
+   raise
waiter = loop.create_future()
-   loop.add_writer(fd, lambda: waiter.set_result(None))
+   loop.add_writer(fd, lambda: waiter.done() or

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2020-02-28 Thread Zac Medico 
commit: 64b11fe4dbcd7f2b4c36d8c40a09425a2c624c7a
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Feb 29 04:24:19 2020 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Feb 29 04:31:50 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=64b11fe4

asyncio: improve _AsyncioEventLoop isinstance logic

Since _AsyncioEventLoop can be wrapped, use the _asyncio_wrapper
attributre for isinstance checks (_wrap_loop guarantees that this
attribute exists).

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/__init__.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index 7635dbb5e..f4b03891f 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = (
@@ -139,7 +139,7 @@ def create_subprocess_exec(*args, **kwargs):
"""
loop = _wrap_loop(kwargs.pop('loop', None))
kwargs.setdefault('close_fds', _close_fds_default)
-   if _asyncio_enabled and isinstance(loop, _AsyncioEventLoop):
+   if _asyncio_enabled and isinstance(loop._asyncio_wrapper, 
_AsyncioEventLoop):
# Use the real asyncio create_subprocess_exec (loop argument
# is deprecated since since Python 3.8).
return _real_asyncio.create_subprocess_exec(*args, **kwargs)
@@ -191,10 +191,10 @@ def ensure_future(coro_or_future, loop=None):
@return: an instance of Future
"""
loop = _wrap_loop(loop)
-   if _asyncio_enabled and isinstance(loop, _AsyncioEventLoop):
+   if _asyncio_enabled and isinstance(loop._asyncio_wrapper, 
_AsyncioEventLoop):
# Use the real asyncio loop and ensure_future.
return _real_asyncio.ensure_future(
-   coro_or_future, loop=loop._loop)
+   coro_or_future, loop=loop._asyncio_wrapper._loop)
 
if isinstance(coro_or_future, Future):
return coro_or_future

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2019-10-17 Thread Zac Medico 
commit: 67e0bba35f1b7afd5ee5ca648154838741875d6a
Author: Zac Medico  gentoo  org>
AuthorDate: Fri Oct 18 03:35:46 2019 +
Commit: Zac Medico  gentoo  org>
CommitDate: Fri Oct 18 03:37:40 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=67e0bba3

asyncio.create_subprocess_exec: suppress DeprecationWarning for loop argument

See: https://bugs.python.org/issue36373
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index bd9f2e47c..7635dbb5e 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -140,8 +140,9 @@ def create_subprocess_exec(*args, **kwargs):
loop = _wrap_loop(kwargs.pop('loop', None))
kwargs.setdefault('close_fds', _close_fds_default)
if _asyncio_enabled and isinstance(loop, _AsyncioEventLoop):
-   # Use the real asyncio loop and create_subprocess_exec.
-   return _real_asyncio.create_subprocess_exec(*args, 
loop=loop._loop, **kwargs)
+   # Use the real asyncio create_subprocess_exec (loop argument
+   # is deprecated since since Python 3.8).
+   return _real_asyncio.create_subprocess_exec(*args, **kwargs)
 
result = loop.create_future()

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/

2019-05-18 Thread Zac Medico 
commit: cef213c31ce05920fd1b8bbf2749f012944f0c2a
Author: Zac Medico  gentoo  org>
AuthorDate: Sat May 18 22:16:13 2019 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat May 18 22:20:50 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cef213c3

_safe_loop: call global_event_loop() directly

Since _wrap_loop() always returns an identical result in this context,
call global_event_loop() directly.

Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index e77c7a690..bd9f2e47c 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -265,6 +265,6 @@ def _safe_loop():
@return: event loop instance
"""
if portage._internal_caller:
-   return _wrap_loop()
+   return _global_event_loop()
else:
return _EventLoop(main=False)

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/util/futures/

2018-09-24 Thread Zac Medico 
commit: 48c06e489e695321e8059da2dac1c03f6624d2e8
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Aug  6 06:43:41 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Sep 24 03:41:17 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=48c06e48

Implement asyncio.iscoroutinefunction for compat_coroutine

Sometimes it's useful to test if a function is a coroutine function,
so implement a version of asyncio.iscoroutinefunction that works
with asyncio.coroutine as well as compat_coroutine.coroutine (since
both kinds of coroutine functions behave identically for our
purposes).

Reviewed-by: Brian Dolbec  gentoo.org>
Signed-off-by: Zac Medico  gentoo.org>

 lib/portage/util/futures/_asyncio/__init__.py | 14 ++
 lib/portage/util/futures/compat_coroutine.py  | 12 
 2 files changed, 26 insertions(+)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py 
b/lib/portage/util/futures/_asyncio/__init__.py
index faab98e47..2a637624d 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -36,6 +36,7 @@ except ImportError:
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures.unix_events:_PortageEventLoopPolicy',
+   'portage.util.futures:compat_coroutine@_compat_coroutine',
 )
 from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as 
_AsyncioEventLoop
 from portage.util._eventloop.global_event_loop import (
@@ -152,6 +153,19 @@ def create_subprocess_exec(*args, **kwargs):
return result
 
 
+def iscoroutinefunction(func):
+   """
+   Return True if func is a decorated coroutine function,
+   supporting both asyncio.coroutine and compat_coroutine since
+   their behavior is identical for all practical purposes.
+   """
+   if _compat_coroutine._iscoroutinefunction(func):
+   return True
+   elif _real_asyncio is not None and 
_real_asyncio.iscoroutinefunction(func):
+   return True
+   return False
+
+
 class Task(Future):
"""
Schedule the execution of a coroutine: wrap it in a future. A task

diff --git a/lib/portage/util/futures/compat_coroutine.py 
b/lib/portage/util/futures/compat_coroutine.py
index 3edfa6bee..b5ff92faf 100644
--- a/lib/portage/util/futures/compat_coroutine.py
+++ b/lib/portage/util/futures/compat_coroutine.py
@@ -8,6 +8,17 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures:asyncio',
 )
 
+# A marker for iscoroutinefunction.
+_is_coroutine = object()
+
+
+def _iscoroutinefunction(func):
+   """
+   Return True if func is a decorated coroutine function
+   created with the coroutine decorator for this module.
+   """
+   return getattr(func, '_is_coroutine', None) is _is_coroutine
+
 
 def coroutine(generator_func):
"""
@@ -34,6 +45,7 @@ def coroutine(generator_func):
@functools.wraps(generator_func)
def wrapped(*args, **kwargs):
return _generator_future(generator_func, *args, **kwargs)
+   wrapped._is_coroutine = _is_coroutine
return wrapped

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/util/futures/, ...

2018-08-05 Thread Zac Medico 
commit: 6b4252d3a0f12808a5bcce888b7f68e1f84b5301
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Jul 28 21:22:42 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Aug  6 04:38:41 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6b4252d3

Add asyncio.create_subprocess_exec support for python2 (bug 662388)

The asyncio.create_subprocess_exec function is essential for
using subprocesses in coroutines, so add support to do this
for python2. This paves the way for extensive use of coroutines
in portage, since coroutines are well-suited for many portage
tasks that involve subprocesses.

Bug: https://bugs.gentoo.org/662388

 .../util/futures/asyncio/test_subprocess_exec.py   | 184 ++---
 lib/portage/util/futures/_asyncio/__init__.py  |  53 ++
 lib/portage/util/futures/_asyncio/process.py   | 107 
 lib/portage/util/futures/_asyncio/streams.py   |  96 +++
 lib/portage/util/futures/compat_coroutine.py   |   6 +-
 5 files changed, 315 insertions(+), 131 deletions(-)

diff --git a/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py 
b/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
index 5a812ba6a..61646cb92 100644
--- a/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
+++ b/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py
@@ -3,61 +3,16 @@
 
 import os
 import subprocess
-
-try:
-   from asyncio import create_subprocess_exec
-except ImportError:
-   create_subprocess_exec = None
+import sys
 
 from portage.process import find_binary
 from portage.tests import TestCase
 from portage.util._eventloop.global_event_loop import global_event_loop
 from portage.util.futures import asyncio
-from portage.util.futures.executor.fork import ForkExecutor
+from portage.util.futures._asyncio import create_subprocess_exec
+from portage.util.futures._asyncio.streams import _reader as reader
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
 from portage.util.futures.unix_events import DefaultEventLoopPolicy
-from _emerge.PipeReader import PipeReader
-
-
-def reader(input_file, loop=None):
-   """
-   Asynchronously read a binary input file.
-
-   @param input_file: binary input file
-   @type input_file: file
-   @param loop: event loop
-   @type loop: EventLoop
-   @return: bytes
-   @rtype: asyncio.Future (or compatible)
-   """
-   loop = asyncio._wrap_loop(loop)
-   future = loop.create_future()
-   _Reader(future, input_file, loop)
-   return future
-
-
-class _Reader(object):
-   def __init__(self, future, input_file, loop):
-   self._future = future
-   self._pipe_reader = PipeReader(
-   input_files={'input_file':input_file}, scheduler=loop)
-
-   self._future.add_done_callback(self._cancel_callback)
-   self._pipe_reader.addExitListener(self._eof)
-   self._pipe_reader.start()
-
-   def _cancel_callback(self, future):
-   if future.cancelled():
-   self._cancel()
-
-   def _eof(self, pipe_reader):
-   self._pipe_reader = None
-   self._future.set_result(pipe_reader.getvalue())
-
-   def _cancel(self):
-   if self._pipe_reader is not None and self._pipe_reader.poll() 
is None:
-   self._pipe_reader.removeExitListener(self._eof)
-   self._pipe_reader.cancel()
-   self._pipe_reader = None
 
 
 class SubprocessExecTestCase(TestCase):
@@ -76,99 +31,66 @@ class SubprocessExecTestCase(TestCase):

self.assertFalse(global_event_loop().is_closed())
 
def testEcho(self):
-   if create_subprocess_exec is None:
-   self.skipTest('create_subprocess_exec not implemented 
for python2')
-
args_tuple = (b'hello', b'world')
echo_binary = find_binary("echo")
self.assertNotEqual(echo_binary, None)
echo_binary = echo_binary.encode()
 
-   # Use os.pipe(), since this loop does not implement the
-   # ReadTransport necessary for subprocess.PIPE support.
-   stdout_pr, stdout_pw = os.pipe()
-   stdout_pr = os.fdopen(stdout_pr, 'rb', 0)
-   stdout_pw = os.fdopen(stdout_pw, 'wb', 0)
-   files = [stdout_pr, stdout_pw]
-
def test(loop):
-   output = None
-   try:
-   with open(os.devnull, 'rb', 0) as devnull:
-   proc = loop.run_until_complete(
-   create_subprocess_exec(
-   echo_binary, *args_tuple,
-   stdin=devnull,