[gentoo-portage-dev] [PATCH 2/2] AsynchronousTask: add async_wait() method (bug 653856)

[Zac Medico]

Since the AsynchronousTask.wait() method is prone to event loop
recursion, deprecate it, and add an async_wait() method method to
replace it. Instead of using task.wait() in order to implicitly run
the event loop, now loop.run_until_complete(task.async_wait()) will
be used to explicitly run the event loop. This explicit approach will
make it more obvious when code will trigger event loop recursion
which would not be compatible with asyncio's default event loop.

Bug: https://bugs.gentoo.org/653856
---
 pym/_emerge/AsynchronousTask.py | 23 +++
 pym/portage/tests/ebuild/test_ipc_daemon.py |  2 +-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/pym/_emerge/AsynchronousTask.py b/pym/_emerge/AsynchronousTask.py
index e29324440..a2f1798fe 100644
--- a/pym/_emerge/AsynchronousTask.py
+++ b/pym/_emerge/AsynchronousTask.py
@@ -29,6 +29,26 @@ class AsynchronousTask(SlotObject):
self._start_hook()
self._start()
 
+   def async_wait(self):
+   """
+   Wait for returncode asynchronously. Notification is available
+   via the add_done_callback method of the returned Future 
instance.
+
+   @returns: Future, result is self.returncode
+   """
+   waiter = self.scheduler.create_future()
+   exit_listener = lambda self: waiter.set_result(self.returncode)
+   self.addExitListener(exit_listener)
+   waiter.add_done_callback(lambda waiter:
+   self.removeExitListener(exit_listener) if 
waiter.cancelled() else None)
+   if self.returncode is not None:
+   # If the returncode is None, it means the exit event 
has already
+   # happened, so use _async_wait() to guarantee that the 
exit_listener
+   # is called. This does not do any harm because a given 
exit
+   # listener is never called more than once.
+   self._async_wait()
+   return waiter
+
def _start(self):
self.returncode = os.EX_OK
self.wait()
@@ -47,6 +67,9 @@ class AsynchronousTask(SlotObject):
return self.returncode
 
def wait(self):
+   """
+   Deprecated. Use async_wait() instead.
+   """
if self.returncode is None:
if not self._waiting:
self._waiting = True
diff --git a/pym/portage/tests/ebuild/test_ipc_daemon.py 
b/pym/portage/tests/ebuild/test_ipc_daemon.py
index bc18cdf64..e6da51a76 100644
--- a/pym/portage/tests/ebuild/test_ipc_daemon.py
+++ b/pym/portage/tests/ebuild/test_ipc_daemon.py
@@ -157,6 +157,6 @@ class IpcDaemonTestCase(TestCase):
try:
task_scheduler.start()
event_loop.run_until_complete(self._run_done)
-   task_scheduler.wait()
+   
event_loop.run_until_complete(task_scheduler.async_wait())
finally:
timeout_handle.cancel()
-- 
2.13.6

[gentoo-portage-dev] [PATCH 1/2] AsynchronousTask: add scheduler attribute (bug 653856)

[Zac Medico]

Add an AsynchronousTask.scheduler attribute, which will hold an event
loop instance in order to work with Future instances. Name the
attribute "scheduler" because this is the name used by many existing
subclasses of AsynchronousTask. Since the AsyncScheduler class already
has an _event_loop attribute that it inherits from PollScheduler, use
a property for compatibility. Also update the SlotObject constructor
to allow a subclass to override an attribute from __slots__ with a
property, so that AsyncScheduler can use a property for the scheduler
attribute.

Bug: https://bugs.gentoo.org/653856
---
 pym/_emerge/AbstractPollTask.py| 3 +--
 pym/_emerge/AsynchronousLock.py| 2 +-
 pym/_emerge/AsynchronousTask.py| 2 +-
 pym/_emerge/CompositeTask.py   | 2 +-
 pym/portage/util/SlotObject.py | 9 -
 pym/portage/util/_async/AsyncScheduler.py  | 7 +++
 pym/portage/util/_async/AsyncTaskFuture.py | 2 +-
 7 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py
index 0d38bd481..aa8fc6593 100644
--- a/pym/_emerge/AbstractPollTask.py
+++ b/pym/_emerge/AbstractPollTask.py
@@ -11,8 +11,7 @@ from _emerge.AsynchronousTask import AsynchronousTask
 
 class AbstractPollTask(AsynchronousTask):
 
-   __slots__ = ("scheduler",) + \
-   ("_registered",)
+   __slots__ = ("_registered",)
 
_bufsize = 4096
 
diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py
index 78ab37ecb..0178feab2 100644
--- a/pym/_emerge/AsynchronousLock.py
+++ b/pym/_emerge/AsynchronousLock.py
@@ -37,7 +37,7 @@ class AsynchronousLock(AsynchronousTask):
signals to the main thread).
"""
 
-   __slots__ = ('path', 'scheduler',) + \
+   __slots__ = ('path',) + \
('_imp', '_force_async', '_force_dummy', '_force_process', \
'_force_thread', '_unlock_future')
 
diff --git a/pym/_emerge/AsynchronousTask.py b/pym/_emerge/AsynchronousTask.py
index da58261db..e29324440 100644
--- a/pym/_emerge/AsynchronousTask.py
+++ b/pym/_emerge/AsynchronousTask.py
@@ -16,7 +16,7 @@ class AsynchronousTask(SlotObject):
the task is complete and self.returncode has been set.
"""
 
-   __slots__ = ("background", "cancelled", "returncode") + \
+   __slots__ = ("background", "cancelled", "returncode", "scheduler") + \
("_exit_listeners", "_exit_listener_stack", "_start_listeners",
"_waiting")
 
diff --git a/pym/_emerge/CompositeTask.py b/pym/_emerge/CompositeTask.py
index f3acc9696..bfd4bacbd 100644
--- a/pym/_emerge/CompositeTask.py
+++ b/pym/_emerge/CompositeTask.py
@@ -6,7 +6,7 @@ from portage import os
 
 class CompositeTask(AsynchronousTask):
 
-   __slots__ = ("scheduler",) + ("_current_task",)
+   __slots__ = ("_current_task",)
 
_TASK_QUEUED = -1
 
diff --git a/pym/portage/util/SlotObject.py b/pym/portage/util/SlotObject.py
index 4bb682258..ba6215874 100644
--- a/pym/portage/util/SlotObject.py
+++ b/pym/portage/util/SlotObject.py
@@ -20,7 +20,14 @@ class SlotObject(object):
raise AssertionError(
"class '%s' duplicates '%s' 
value in __slots__ of base class '%s'" %
(self.__class__.__name__, 
myattr, c.__name__))
-   setattr(self, myattr, myvalue)
+   try:
+   setattr(self, myattr, myvalue)
+   except AttributeError:
+   # Allow a property to override a 
__slots__ value, but raise an
+   # error if the intended value is 
something other than None.
+   if not (myvalue is None and
+   isinstance(getattr(type(self), 
myattr, None), property)):
+   raise
 
if kwargs:
raise TypeError(
diff --git a/pym/portage/util/_async/AsyncScheduler.py 
b/pym/portage/util/_async/AsyncScheduler.py
index 9beb8a848..b89b57dab 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -20,6 +20,13 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
self._remaining_tasks = True
self._loadavg_check_id = None
 
+   @property
+   def scheduler(self):
+   """
+   Provides compatibility with the AsynchronousTask.scheduler 
attribute.
+   """
+   return self._event_loop
+
def _poll(self):
if not (self._is_work_scheduled() or self._keep_scheduling()):
self.wait()
diff --git a/pym/portage/util/_async/AsyncTaskFuture.py 

[gentoo-portage-dev] [PATCH 0/2] AsynchronousTask: add async_wait() method (bug 653856)

[Zac Medico]

Since the AsynchronousTask.wait() method is prone to event loop
recursion, deprecate it, and add an async_wait() method method to
replace it. Instead of using task.wait() in order to implicitly run
the event loop, now loop.run_until_complete(task.async_wait()) will
be used to explicitly run the event loop. This explicit approach will
make it more obvious when code will trigger event loop recursion
which would not be compatible with asyncio's default event loop.

Bug: https://bugs.gentoo.org/653856

Zac Medico (2):
  AsynchronousTask: add scheduler attribute (bug 653856)
  AsynchronousTask: add async_wait() method (bug 653856)

 pym/_emerge/AbstractPollTask.py |  3 +--
 pym/_emerge/AsynchronousLock.py |  2 +-
 pym/_emerge/AsynchronousTask.py | 25 -
 pym/_emerge/CompositeTask.py|  2 +-
 pym/portage/tests/ebuild/test_ipc_daemon.py |  2 +-
 pym/portage/util/SlotObject.py  |  9 -
 pym/portage/util/_async/AsyncScheduler.py   |  7 +++
 pym/portage/util/_async/AsyncTaskFuture.py  |  2 +-
 8 files changed, 44 insertions(+), 8 deletions(-)

-- 
2.13.6

Re: [gentoo-portage-dev] [PATCH 0/5] EbuildFetcher._get_uri_map(): fix event loop recursion (bug 653810)

[Zac Medico]

On 04/23/2018 09:10 AM, Brian Dolbec wrote:
> On Sun, 22 Apr 2018 15:30:09 -0700
> Zac Medico  wrote:
> 
>> Bug: https://bugs.gentoo.org/653810
>>
>> Zac Medico (5):
>>   portdbapi: add async_fetch_map method (bug 653810)
>>   EbuildFetcher: inherit CompositeTask (bug 653810)
>>   EbuildFetcher: add _async_uri_map method (bug 653810)
>>   EbuildFetcher: use _async_uri_map in _start (bug 653810)
>>   EbuildFetcher: add async_already_fetched method (bug 653810)
>>
>>  pym/_emerge/EbuildBuild.py|   8 +++-
>>  pym/_emerge/EbuildFetcher.py  | 105
>> --
>> pym/portage/dbapi/porttree.py |  75 +++--- 3
>> files changed, 146 insertions(+), 42 deletions(-)
>>
> 
> I didn't see any errors glaring at me...  :)
> 
> looks like it should be good 

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=0a5692bd4f4ab4a54daf2ce09112617cbc21c9ad
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 0/5] EbuildFetcher._get_uri_map(): fix event loop recursion (bug 653810)

[Brian Dolbec]

On Sun, 22 Apr 2018 15:30:09 -0700
Zac Medico  wrote:

> Bug: https://bugs.gentoo.org/653810
> 
> Zac Medico (5):
>   portdbapi: add async_fetch_map method (bug 653810)
>   EbuildFetcher: inherit CompositeTask (bug 653810)
>   EbuildFetcher: add _async_uri_map method (bug 653810)
>   EbuildFetcher: use _async_uri_map in _start (bug 653810)
>   EbuildFetcher: add async_already_fetched method (bug 653810)
> 
>  pym/_emerge/EbuildBuild.py|   8 +++-
>  pym/_emerge/EbuildFetcher.py  | 105
> --
> pym/portage/dbapi/porttree.py |  75 +++--- 3
> files changed, 146 insertions(+), 42 deletions(-)
> 

I didn't see any errors glaring at me...  :)

looks like it should be good 

-- 
Brian Dolbec