dabla commented on code in PR #76545:
URL: https://github.com/apache/airflow/pull/76545#discussion_r4614256095


##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -312,42 +339,43 @@ def send(self, msg: SendMsgType) -> ReceiveMsgType | None:
 
             return self._get_response()
         finally:
-            self._thread_lock.release()
+            self._exchanging = False
+
+    def _locked_exchange(

Review Comment:
   Renamed in 54a00819cf.



##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -245,55 +271,56 @@ class CommsDecoder(Generic[ReceiveMsgType, SendMsgType]):
 
     err_decoder: TypeAdapter[ErrorResponse] = attrs.field(factory=lambda: 
TypeAdapter(ToTask), repr=False)
 
-    _thread_lock: threading.Lock = attrs.field(factory=threading.Lock, 
repr=False)
-    _async_lock: asyncio.Lock = attrs.field(factory=asyncio.Lock, repr=False)
-    _loop_thread_id: int | None = attrs.field(default=None, repr=False, 
init=False)
+    # Reentrant, so that a send() which interrupts its own thread's (a signal 
handler on the main
+    # thread) is let in instead of waiting for itself, and can tell from 
_exchanging whether that
+    # thread was in the middle of a request. The RLock records its owner when 
it is taken, so no
+    # moment is left where the thread holds the lock but is not yet known as 
its holder.
+    _thread_lock: threading.RLock = attrs.field(factory=threading.RLock, 
repr=False)
+    _worker: _RequestWorker = attrs.field(factory=lambda: _RequestWorker(), 
repr=False, init=False)
+    # Whether the thread holding _thread_lock is between writing a request and 
reading its
+    # response. Only that thread can read it as True once it holds the lock 
itself.
+    _exchanging: bool = attrs.field(default=False, repr=False, init=False)
 
     def _make_frame(self, msg: SendMsgType) -> _RequestFrame:
         carrier: dict[str, str] = {}
         _trace_propagator.inject(carrier)
         return _RequestFrame(id=next(self.id_counter), body=msg.model_dump(), 
context_carrier=carrier or None)
 
-    @property
-    def _is_on_loop_thread(self) -> bool:
-        if threading.get_ident() == self._loop_thread_id:
-            with suppress(RuntimeError):
-                return bool(asyncio.get_running_loop())
-        return False
-
-    @property
-    def _asend_in_flight_on_this_thread(self) -> bool:
+    def send(self, msg: SendMsgType) -> ReceiveMsgType | None:
         """
-        Whether an ``asend()`` coroutine of a loop that runs on the current 
thread is in flight.
-
-        ``asend()`` holds ``_async_lock`` for its whole duration, from before 
it takes
-        ``_thread_lock`` (in an executor thread) until after it releases it, 
so the async lock
-        being held is the reliable signal that the thread lock is, or is about 
to be, held by a
-        coroutine. Such a coroutine only makes progress when its loop runs, 
and that loop runs on
-        this thread. This is thread-scoped on purpose: a loop running on 
*another* thread keeps
-        going while this one blocks, so ``send()`` from there can safely wait 
for the lock.
+        Send a request to the parent and block until the response is received.
+
+        Called on an event loop's thread while an ``asend()`` is in flight, 
this waits for that
+        request's round trip: ``asend()`` runs on a worker thread that needs 
no loop to finish, so
+        the wait ends, but it stalls every coroutine on the loop meanwhile. 
Async code should
+        prefer ``asend()`` and the async SDK calls built on it. Only a call on 
a thread that is
+        itself in the middle of a request raises 
:class:`DeadlockImminentError`, as waiting there
+        would never end.
         """
-        return threading.get_ident() == self._loop_thread_id and 
self._async_lock.locked()
-
-    def send(self, msg: SendMsgType) -> ReceiveMsgType | None:
-        """Send a request to the parent and block until the response is 
received."""
         frame_bytes = self._make_frame(msg).as_bytes()
-
-        # An asend() in flight on this thread's loop can only release 
_thread_lock once that loop
-        # runs again, and the loop cannot run while this thread blocks in 
send(). Raise instead of
-        # blocking. This covers the loop being paused between two 
run_until_complete() calls, where
-        # the running-loop check below does not apply: 
asyncio.get_running_loop() raises, so send()
-        # would otherwise fall through to a blocking acquire and freeze the 
process with every
-        # thread idle (seen with a sync XCom read pulling the next iterated 
sub-task's input while a
-        # sub-task's asend() was parked mid-I/O).
-        if self._asend_in_flight_on_this_thread:
-            raise DeadlockImminentError(msg)
-
-        # When called from the event loop thread while the loop is running, 
use non-blocking
-        # acquire to detect an imminent deadlock: an asend() coroutine 
currently holds
-        # _thread_lock and is waiting for the event loop to complete its I/O.
-        if not self._thread_lock.acquire(blocking=not self._is_on_loop_thread):
-            raise DeadlockImminentError(msg)
+        with self._thread_lock:
+            # Another thread's request makes the acquire above wait for it. 
This thread's own lets
+            # it in again: in the middle of that request, the call would wait 
for its own response,
+            # so it raises; before the request is written or after its 
response is read, the
+            # channel is free and this call is exchanged whole in between.
+            if self._exchanging:
+                raise DeadlockImminentError(msg)
+            return self._exchange(msg, frame_bytes)
+
+    def _exchange(self, msg: SendMsgType, frame_bytes: bytes | bytearray) -> 
ReceiveMsgType | None:
+        """
+        Write one request and read its response; the caller holds 
``_thread_lock``.
+
+        Responses are read in order: :meth:`_get_response` takes the next 
frame and does not check
+        its id. An exception that unwinds this between the write and the read 
therefore leaves the
+        response on the socket, and the next request reads it as its own. That 
happens when a
+        signal handler interrupts the request on the main thread and raises 
into it: the
+        ``DeadlockImminentError`` of a handler that calls :meth:`send` itself, 
or the
+        ``AirflowTaskTimeout`` of the ``execution_timeout`` alarm, which any 
task can hit. Making
+        the channel safe after that needs a read that resumes a partly read 
frame, or a channel
+        marked unusable, and is left to a change of its own.
+        """

Review Comment:
   Opened #77240 and shortened the docstring to a pointer to it in 54a00819cf. 
The fix touches every request (an id check, or a channel marked unusable after 
an interrupted exchange), so it gets a PR of its own.



##########
task-sdk/tests/task_sdk/execution_time/test_comms.py:
##########
@@ -356,64 +468,313 @@ async def _async_phase():
 
         assert result is not None
 
-    def 
test_send_from_paused_event_loop_thread_raises_when_asend_in_flight(self, 
socket_pair):
+    @staticmethod
+    def _serve(sock, count: int, delay: float = 0.0) -> None:
+        """Answer ``count`` requests with their own key, after ``delay`` 
seconds each; stop on a closed socket."""
+        import time

Review Comment:
   Moved to the top in 54a00819cf. The two redundant `import asyncio` inside 
tests in this file are gone too.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to