amoghrajesh commented on code in PR #76545:
URL: https://github.com/apache/airflow/pull/76545#discussion_r4611117884
##########
task-sdk/tests/task_sdk/execution_time/test_comms.py:
##########
@@ -356,64 +468,313 @@ async def _async_phase():
assert result is not None
- def
test_send_from_paused_event_loop_thread_raises_when_asend_in_flight(self,
socket_pair):
+ @staticmethod
+ def _serve(sock, count: int, delay: float = 0.0) -> None:
+ """Answer ``count`` requests with their own key, after ``delay``
seconds each; stop on a closed socket."""
+ import time
+
+ try:
+ for _ in range(count):
+ length = int.from_bytes(sock.recv(4), "big")
+ body = b""
+ while len(body) < length:
+ body += sock.recv(length - len(body))
+ request = msgspec.msgpack.decode(body, type=_RequestFrame)
+ assert request.body is not None
+ time.sleep(delay)
+ resp = {"type": "VariableResult", "key": request.body["key"],
"value": "v"}
+ encoded = msgspec.msgpack.encode(_ResponseFrame(request.id,
resp, None))
+ sock.sendall(len(encoded).to_bytes(4, "big") + encoded)
+ except OSError:
+ pass
+
+ def
test_send_on_a_thread_already_in_a_request_raises_instead_of_waiting(self,
socket_pair):
+ """
+ A call on a thread that holds the channel for its own request would
wait for itself: the
+ one real deadlock left, so it raises at once, and the interrupted
request still completes.
"""
- Regression: send() called from the loop's own thread while the loop is
*paused* (between
- two run_until_complete() calls) and an asend() is parked mid-I/O must
raise
- DeadlockImminentError instead of blocking forever.
+ r, w = socket_pair
+ decoder = CommsDecoder(socket=r, log=structlog.get_logger())
+ threading.Thread(target=self._serve, args=(w, 1), daemon=True).start()
+ raised: list[BaseException] = []
+ original_get_response = CommsDecoder._get_response
+
+ def interrupted(self):
+ # What a signal handler would do while this thread waits for the
response.
+ try:
+ self.send(GetVariable(key="from the handler"))
+ except DeadlockImminentError as exc:
+ raised.append(exc)
+ return original_get_response(self)
+
+ with mock.patch.object(CommsDecoder, "_get_response", interrupted):
+ result = decoder.send(GetVariable(key="outer"))
+
+ assert result.key == "outer"
+ assert len(raised) == 1
+ assert "already in the middle of a request" in str(raised[0])
+ assert not _is_taken(decoder._thread_lock)
- The running-loop check alone does not cover this:
asyncio.get_running_loop() raises while
- the loop is paused, so send() fell through to a blocking acquire of
_thread_lock. The
- holder, an asend() coroutine, can only release it once the loop runs
again, on this very
- thread, which is now blocked. Seen in production as an
IterableOperator freezing with
- every thread idle: AsyncAwareExecutor.map pulled the next sub-task's
input through a sync
- XCom read between two run_until_complete() calls while a sub-task's
asend() was in flight.
+ @pytest.mark.skipif(not hasattr(signal, "setitimer"), reason="needs
SIGALRM timers")
+ def
test_a_signal_handler_calling_send_during_a_send_raises_instead_of_freezing(self,
socket_pair):
+ """
+ The case the check exists for: the runner's SIGTERM handler runs
on_kill() on the main
+ thread, and an on_kill() making a sync SDK call while the main thread
waits for a response
+ would wait for its own thread. A real signal lands mid-request here.
"""
r, w = socket_pair
decoder = CommsDecoder(socket=r, log=structlog.get_logger())
+ # The response comes after the signal has fired.
+ threading.Thread(target=self._serve, args=(w, 1, 0.3),
daemon=True).start()
+ raised: list[BaseException] = []
- def _read_request(sock) -> _RequestFrame:
- length = int.from_bytes(sock.recv(4), "big")
- body = b""
- while len(body) < length:
- body += sock.recv(length - len(body))
- return msgspec.msgpack.decode(body, type=_RequestFrame)
+ def on_alarm(signum, frame):
+ try:
+ decoder.send(GetVariable(key="from on_kill"))
+ except DeadlockImminentError as exc:
+ raised.append(exc)
- def _respond(sock, req: _RequestFrame) -> None:
- assert req.body is not None
- resp = {"type": "VariableResult", "key": req.body["key"], "value":
"v"}
- encoded = msgspec.msgpack.encode(_ResponseFrame(req.id, resp,
None))
- sock.sendall(len(encoded).to_bytes(4, "big") + encoded)
+ previous = signal.signal(signal.SIGALRM, on_alarm)
+ try:
+ signal.setitimer(signal.ITIMER_REAL, 0.05)
+ result = decoder.send(GetVariable(key="interrupted"))
+ finally:
+ signal.setitimer(signal.ITIMER_REAL, 0)
+ signal.signal(signal.SIGALRM, previous)
+
+ assert result.key == "interrupted"
+ assert len(raised) == 1
+
+ def test_a_failed_request_leaves_no_request_in_flight_behind(self,
socket_pair):
+ """
+ A request that fails mid-exchange releases the channel and clears its
in-flight mark, so the
+ thread's next call is served instead of being taken for a call from
inside a request.
+ """
+ r, w = socket_pair
+ decoder = CommsDecoder(socket=r, log=structlog.get_logger())
+ threading.Thread(target=self._serve, args=(w, 2), daemon=True).start()
+
+ with mock.patch.object(
+ CommsDecoder, "_get_response", autospec=True,
side_effect=EOFError("supervisor gone")
+ ):
+ with pytest.raises(EOFError):
+ decoder.send(GetVariable(key="lost"))
+
+ assert decoder._exchanging is False
+ assert not _is_taken(decoder._thread_lock)
+ assert decoder.send(GetVariable(key="next")) is not None
Review Comment:
The failed-request test passes for the wrong reason. The next call reads the
old "lost" answer, and is `not None` still passes. Check `.key == "next"`
instead.
##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -225,6 +223,34 @@ class _ResponseFrame(_FrameMixin, msgspec.Struct,
array_like=True, frozen=True,
error: dict[str, Any] | None = None
+class _RequestWorker:
+ """
+ The one thread that makes a decoder's ``asend()`` requests, whichever
event loop they come from.
+
+ Requests queue for it in the order they were made, and it needs no loop to
finish one. The
+ first request of a process starts it: threads do not survive a fork, so a
forked child gets
+ a thread of its own.
+ """
+
+ __slots__ = ("_executor", "_lock", "_pid")
+
+ def __init__(self) -> None:
+ self._executor: ThreadPoolExecutor | None = None
+ self._pid: int | None = None
+ self._lock = threading.Lock()
+
+ def executor(self) -> ThreadPoolExecutor:
Review Comment:
Can we call this `get_executor`?
##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -245,55 +271,56 @@ class CommsDecoder(Generic[ReceiveMsgType, SendMsgType]):
err_decoder: TypeAdapter[ErrorResponse] = attrs.field(factory=lambda:
TypeAdapter(ToTask), repr=False)
- _thread_lock: threading.Lock = attrs.field(factory=threading.Lock,
repr=False)
- _async_lock: asyncio.Lock = attrs.field(factory=asyncio.Lock, repr=False)
- _loop_thread_id: int | None = attrs.field(default=None, repr=False,
init=False)
+ # Reentrant, so that a send() which interrupts its own thread's (a signal
handler on the main
+ # thread) is let in instead of waiting for itself, and can tell from
_exchanging whether that
+ # thread was in the middle of a request. The RLock records its owner when
it is taken, so no
+ # moment is left where the thread holds the lock but is not yet known as
its holder.
+ _thread_lock: threading.RLock = attrs.field(factory=threading.RLock,
repr=False)
+ _worker: _RequestWorker = attrs.field(factory=lambda: _RequestWorker(),
repr=False, init=False)
Review Comment:
Why not just `factory=_RequestWorker`?
##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -245,55 +271,56 @@ class CommsDecoder(Generic[ReceiveMsgType, SendMsgType]):
err_decoder: TypeAdapter[ErrorResponse] = attrs.field(factory=lambda:
TypeAdapter(ToTask), repr=False)
- _thread_lock: threading.Lock = attrs.field(factory=threading.Lock,
repr=False)
- _async_lock: asyncio.Lock = attrs.field(factory=asyncio.Lock, repr=False)
- _loop_thread_id: int | None = attrs.field(default=None, repr=False,
init=False)
+ # Reentrant, so that a send() which interrupts its own thread's (a signal
handler on the main
+ # thread) is let in instead of waiting for itself, and can tell from
_exchanging whether that
+ # thread was in the middle of a request. The RLock records its owner when
it is taken, so no
+ # moment is left where the thread holds the lock but is not yet known as
its holder.
+ _thread_lock: threading.RLock = attrs.field(factory=threading.RLock,
repr=False)
+ _worker: _RequestWorker = attrs.field(factory=lambda: _RequestWorker(),
repr=False, init=False)
+ # Whether the thread holding _thread_lock is between writing a request and
reading its
+ # response. Only that thread can read it as True once it holds the lock
itself.
+ _exchanging: bool = attrs.field(default=False, repr=False, init=False)
def _make_frame(self, msg: SendMsgType) -> _RequestFrame:
carrier: dict[str, str] = {}
_trace_propagator.inject(carrier)
return _RequestFrame(id=next(self.id_counter), body=msg.model_dump(),
context_carrier=carrier or None)
- @property
- def _is_on_loop_thread(self) -> bool:
- if threading.get_ident() == self._loop_thread_id:
- with suppress(RuntimeError):
- return bool(asyncio.get_running_loop())
- return False
-
- @property
- def _asend_in_flight_on_this_thread(self) -> bool:
+ def send(self, msg: SendMsgType) -> ReceiveMsgType | None:
"""
- Whether an ``asend()`` coroutine of a loop that runs on the current
thread is in flight.
-
- ``asend()`` holds ``_async_lock`` for its whole duration, from before
it takes
- ``_thread_lock`` (in an executor thread) until after it releases it,
so the async lock
- being held is the reliable signal that the thread lock is, or is about
to be, held by a
- coroutine. Such a coroutine only makes progress when its loop runs,
and that loop runs on
- this thread. This is thread-scoped on purpose: a loop running on
*another* thread keeps
- going while this one blocks, so ``send()`` from there can safely wait
for the lock.
+ Send a request to the parent and block until the response is received.
+
+ Called on an event loop's thread while an ``asend()`` is in flight,
this waits for that
+ request's round trip: ``asend()`` runs on a worker thread that needs
no loop to finish, so
+ the wait ends, but it stalls every coroutine on the loop meanwhile.
Async code should
+ prefer ``asend()`` and the async SDK calls built on it. Only a call on
a thread that is
+ itself in the middle of a request raises
:class:`DeadlockImminentError`, as waiting there
+ would never end.
"""
- return threading.get_ident() == self._loop_thread_id and
self._async_lock.locked()
-
- def send(self, msg: SendMsgType) -> ReceiveMsgType | None:
- """Send a request to the parent and block until the response is
received."""
frame_bytes = self._make_frame(msg).as_bytes()
-
- # An asend() in flight on this thread's loop can only release
_thread_lock once that loop
- # runs again, and the loop cannot run while this thread blocks in
send(). Raise instead of
- # blocking. This covers the loop being paused between two
run_until_complete() calls, where
- # the running-loop check below does not apply:
asyncio.get_running_loop() raises, so send()
- # would otherwise fall through to a blocking acquire and freeze the
process with every
- # thread idle (seen with a sync XCom read pulling the next iterated
sub-task's input while a
- # sub-task's asend() was parked mid-I/O).
- if self._asend_in_flight_on_this_thread:
- raise DeadlockImminentError(msg)
-
- # When called from the event loop thread while the loop is running,
use non-blocking
- # acquire to detect an imminent deadlock: an asend() coroutine
currently holds
- # _thread_lock and is waiting for the event loop to complete its I/O.
- if not self._thread_lock.acquire(blocking=not self._is_on_loop_thread):
- raise DeadlockImminentError(msg)
+ with self._thread_lock:
+ # Another thread's request makes the acquire above wait for it.
This thread's own lets
+ # it in again: in the middle of that request, the call would wait
for its own response,
+ # so it raises; before the request is written or after its
response is read, the
+ # channel is free and this call is exchanged whole in between.
+ if self._exchanging:
+ raise DeadlockImminentError(msg)
+ return self._exchange(msg, frame_bytes)
+
+ def _exchange(self, msg: SendMsgType, frame_bytes: bytes | bytearray) ->
ReceiveMsgType | None:
+ """
+ Write one request and read its response; the caller holds
``_thread_lock``.
+
+ Responses are read in order: :meth:`_get_response` takes the next
frame and does not check
+ its id. An exception that unwinds this between the write and the read
therefore leaves the
+ response on the socket, and the next request reads it as its own. That
happens when a
+ signal handler interrupts the request on the main thread and raises
into it: the
+ ``DeadlockImminentError`` of a handler that calls :meth:`send` itself,
or the
+ ``AirflowTaskTimeout`` of the ``execution_timeout`` alarm, which any
task can hit. Making
+ the channel safe after that needs a read that resumes a partly read
frame, or a channel
+ marked unusable, and is left to a change of its own.
+ """
Review Comment:
Why spend 11 lines in the docstring on a known bug it doesn't fix. Maybe
just open an issue and link it (or fix it) and shorten the docstring
##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -245,55 +271,56 @@ class CommsDecoder(Generic[ReceiveMsgType, SendMsgType]):
err_decoder: TypeAdapter[ErrorResponse] = attrs.field(factory=lambda:
TypeAdapter(ToTask), repr=False)
- _thread_lock: threading.Lock = attrs.field(factory=threading.Lock,
repr=False)
- _async_lock: asyncio.Lock = attrs.field(factory=asyncio.Lock, repr=False)
- _loop_thread_id: int | None = attrs.field(default=None, repr=False,
init=False)
+ # Reentrant, so that a send() which interrupts its own thread's (a signal
handler on the main
+ # thread) is let in instead of waiting for itself, and can tell from
_exchanging whether that
+ # thread was in the middle of a request. The RLock records its owner when
it is taken, so no
+ # moment is left where the thread holds the lock but is not yet known as
its holder.
+ _thread_lock: threading.RLock = attrs.field(factory=threading.RLock,
repr=False)
+ _worker: _RequestWorker = attrs.field(factory=lambda: _RequestWorker(),
repr=False, init=False)
+ # Whether the thread holding _thread_lock is between writing a request and
reading its
+ # response. Only that thread can read it as True once it holds the lock
itself.
+ _exchanging: bool = attrs.field(default=False, repr=False, init=False)
Review Comment:
nit: rename to `_is_exchanging`
##########
task-sdk/src/airflow/sdk/execution_time/comms.py:
##########
@@ -312,42 +339,43 @@ def send(self, msg: SendMsgType) -> ReceiveMsgType | None:
return self._get_response()
finally:
- self._thread_lock.release()
+ self._exchanging = False
+
+ def _locked_exchange(
Review Comment:
nit: rename to `_acquire_and_exchange` as it reflects whats done better
##########
task-sdk/tests/task_sdk/execution_time/test_comms.py:
##########
@@ -356,64 +468,313 @@ async def _async_phase():
assert result is not None
- def
test_send_from_paused_event_loop_thread_raises_when_asend_in_flight(self,
socket_pair):
+ @staticmethod
+ def _serve(sock, count: int, delay: float = 0.0) -> None:
+ """Answer ``count`` requests with their own key, after ``delay``
seconds each; stop on a closed socket."""
+ import time
+
+ try:
+ for _ in range(count):
+ length = int.from_bytes(sock.recv(4), "big")
+ body = b""
+ while len(body) < length:
+ body += sock.recv(length - len(body))
+ request = msgspec.msgpack.decode(body, type=_RequestFrame)
+ assert request.body is not None
+ time.sleep(delay)
+ resp = {"type": "VariableResult", "key": request.body["key"],
"value": "v"}
+ encoded = msgspec.msgpack.encode(_ResponseFrame(request.id,
resp, None))
+ sock.sendall(len(encoded).to_bytes(4, "big") + encoded)
+ except OSError:
+ pass
+
+ def
test_send_on_a_thread_already_in_a_request_raises_instead_of_waiting(self,
socket_pair):
Review Comment:
This test and
`test_a_signal_handler_calling_send_during_a_send_raises_instead_of_freezing`
might hang on main instead of failing. Add `execution_timeout(10)`.
##########
task-sdk/tests/task_sdk/execution_time/test_comms.py:
##########
@@ -216,25 +236,74 @@ def _recv_exactly(sock, n):
assert results[idx].key == f"key{idx}", f"Out-of-order or missing
response for thread {idx}"
assert results[idx].value == f"value{idx}", f"Incorrect value for
thread {idx}"
+ @staticmethod
+ def _answer_in_order(sock, first_read: threading.Event, answer_first:
threading.Event) -> None:
Review Comment:
This same frame read and reply code is copied a lot of times: here, line
348, line 472. Can you just pull it into two shared helpers?
##########
task-sdk/tests/task_sdk/execution_time/test_comms.py:
##########
@@ -356,64 +468,313 @@ async def _async_phase():
assert result is not None
- def
test_send_from_paused_event_loop_thread_raises_when_asend_in_flight(self,
socket_pair):
+ @staticmethod
+ def _serve(sock, count: int, delay: float = 0.0) -> None:
+ """Answer ``count`` requests with their own key, after ``delay``
seconds each; stop on a closed socket."""
+ import time
Review Comment:
Top level import pls.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]