dabla commented on code in PR #76545:
URL: https://github.com/apache/airflow/pull/76545#discussion_r4614256095
##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -312,42 +339,43 @@ def send(self, msg: SendMsgType) -> ReceiveMsgType | None:
return self._get_response()
finally:
- self._thread_lock.release()
+ self._exchanging = False
+
+ def _locked_exchange(
Review Comment:
Renamed in 54a00819cf.
##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -245,55 +271,56 @@ class CommsDecoder(Generic[ReceiveMsgType, SendMsgType]):
err_decoder: TypeAdapter[ErrorResponse] = attrs.field(factory=lambda:
TypeAdapter(ToTask), repr=False)
- _thread_lock: threading.Lock = attrs.field(factory=threading.Lock,
repr=False)
- _async_lock: asyncio.Lock = attrs.field(factory=asyncio.Lock, repr=False)
- _loop_thread_id: int | None = attrs.field(default=None, repr=False,
init=False)
+ # Reentrant, so that a send() which interrupts its own thread's (a signal
handler on the main
+ # thread) is let in instead of waiting for itself, and can tell from
_exchanging whether that
+ # thread was in the middle of a request. The RLock records its owner when
it is taken, so no
+ # moment is left where the thread holds the lock but is not yet known as
its holder.
+ _thread_lock: threading.RLock = attrs.field(factory=threading.RLock,
repr=False)
+ _worker: _RequestWorker = attrs.field(factory=lambda: _RequestWorker(),
repr=False, init=False)
+ # Whether the thread holding _thread_lock is between writing a request and
reading its
+ # response. Only that thread can read it as True once it holds the lock
itself.
+ _exchanging: bool = attrs.field(default=False, repr=False, init=False)
def _make_frame(self, msg: SendMsgType) -> _RequestFrame:
carrier: dict[str, str] = {}
_trace_propagator.inject(carrier)
return _RequestFrame(id=next(self.id_counter), body=msg.model_dump(),
context_carrier=carrier or None)
- @property
- def _is_on_loop_thread(self) -> bool:
- if threading.get_ident() == self._loop_thread_id:
- with suppress(RuntimeError):
- return bool(asyncio.get_running_loop())
- return False
-
- @property
- def _asend_in_flight_on_this_thread(self) -> bool:
+ def send(self, msg: SendMsgType) -> ReceiveMsgType | None:
"""
- Whether an ``asend()`` coroutine of a loop that runs on the current
thread is in flight.
-
- ``asend()`` holds ``_async_lock`` for its whole duration, from before
it takes
- ``_thread_lock`` (in an executor thread) until after it releases it,
so the async lock
- being held is the reliable signal that the thread lock is, or is about
to be, held by a
- coroutine. Such a coroutine only makes progress when its loop runs,
and that loop runs on
- this thread. This is thread-scoped on purpose: a loop running on
*another* thread keeps
- going while this one blocks, so ``send()`` from there can safely wait
for the lock.
+ Send a request to the parent and block until the response is received.
+
+ Called on an event loop's thread while an ``asend()`` is in flight,
this waits for that
+ request's round trip: ``asend()`` runs on a worker thread that needs
no loop to finish, so
+ the wait ends, but it stalls every coroutine on the loop meanwhile.
Async code should
+ prefer ``asend()`` and the async SDK calls built on it. Only a call on
a thread that is
+ itself in the middle of a request raises
:class:`DeadlockImminentError`, as waiting there
+ would never end.
"""
- return threading.get_ident() == self._loop_thread_id and
self._async_lock.locked()
-
- def send(self, msg: SendMsgType) -> ReceiveMsgType | None:
- """Send a request to the parent and block until the response is
received."""
frame_bytes = self._make_frame(msg).as_bytes()
-
- # An asend() in flight on this thread's loop can only release
_thread_lock once that loop
- # runs again, and the loop cannot run while this thread blocks in
send(). Raise instead of
- # blocking. This covers the loop being paused between two
run_until_complete() calls, where
- # the running-loop check below does not apply:
asyncio.get_running_loop() raises, so send()
- # would otherwise fall through to a blocking acquire and freeze the
process with every
- # thread idle (seen with a sync XCom read pulling the next iterated
sub-task's input while a
- # sub-task's asend() was parked mid-I/O).
- if self._asend_in_flight_on_this_thread:
- raise DeadlockImminentError(msg)
-
- # When called from the event loop thread while the loop is running,
use non-blocking
- # acquire to detect an imminent deadlock: an asend() coroutine
currently holds
- # _thread_lock and is waiting for the event loop to complete its I/O.
- if not self._thread_lock.acquire(blocking=not self._is_on_loop_thread):
- raise DeadlockImminentError(msg)
+ with self._thread_lock:
+ # Another thread's request makes the acquire above wait for it.
This thread's own lets
+ # it in again: in the middle of that request, the call would wait
for its own response,
+ # so it raises; before the request is written or after its
response is read, the
+ # channel is free and this call is exchanged whole in between.
+ if self._exchanging:
+ raise DeadlockImminentError(msg)
+ return self._exchange(msg, frame_bytes)
+
+ def _exchange(self, msg: SendMsgType, frame_bytes: bytes | bytearray) ->
ReceiveMsgType | None:
+ """
+ Write one request and read its response; the caller holds
``_thread_lock``.
+
+ Responses are read in order: :meth:`_get_response` takes the next
frame and does not check
+ its id. An exception that unwinds this between the write and the read
therefore leaves the
+ response on the socket, and the next request reads it as its own. That
happens when a
+ signal handler interrupts the request on the main thread and raises
into it: the
+ ``DeadlockImminentError`` of a handler that calls :meth:`send` itself,
or the
+ ``AirflowTaskTimeout`` of the ``execution_timeout`` alarm, which any
task can hit. Making
+ the channel safe after that needs a read that resumes a partly read
frame, or a channel
+ marked unusable, and is left to a change of its own.
+ """
Review Comment:
Opened #77240 and shortened the docstring to a pointer to it in 54a00819cf.
The fix touches every request (an id check, or a channel marked unusable after
an interrupted exchange), so it gets a PR of its own.
##########
task-sdk/tests/task_sdk/execution_time/test_comms.py:
##########
@@ -356,64 +468,313 @@ async def _async_phase():
assert result is not None
- def
test_send_from_paused_event_loop_thread_raises_when_asend_in_flight(self,
socket_pair):
+ @staticmethod
+ def _serve(sock, count: int, delay: float = 0.0) -> None:
+ """Answer ``count`` requests with their own key, after ``delay``
seconds each; stop on a closed socket."""
+ import time
Review Comment:
Moved to the top in 54a00819cf. The two redundant `import asyncio` inside
tests in this file are gone too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]