This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b3c477da8bf Ensure that the Execution API's '/{ti}/run' end point is
idempotent. (#47126)
b3c477da8bf is described below
commit b3c477da8bfc328e438022e31c73e9f37ac5f6fa
Author: Ash Berlin-Taylor <[email protected]>
AuthorDate: Thu Feb 27 09:27:03 2025 +0000
Ensure that the Execution API's '/{ti}/run' end point is idempotent.
(#47126)
If there are any network issues between the client and the API server, it
was
possible that the API could receive the request, update the state to
RUNNING,
but the client never gets the response. When this happens the client would
retry the HTTP request, but the retry would fail with a HTTP 409 Conflict
error as the state was already marked as running.
This fixes that by accepting the request again, as long as it is from the
same
host+unixname+pid we saw already.
Co-authored-by: Mehul Goyal <[email protected]>
---
.../execution_api/routes/task_instances.py | 89 +++++++++++-----------
.../execution_api/routes/test_task_instances.py | 31 ++++++++
2 files changed, 76 insertions(+), 44 deletions(-)
diff --git a/airflow/api_fastapi/execution_api/routes/task_instances.py
b/airflow/api_fastapi/execution_api/routes/task_instances.py
index 6a403c1957a..9ba5bfd3fea 100644
--- a/airflow/api_fastapi/execution_api/routes/task_instances.py
+++ b/airflow/api_fastapi/execution_api/routes/task_instances.py
@@ -49,7 +49,7 @@ from airflow.models.taskreschedule import TaskReschedule
from airflow.models.trigger import Trigger
from airflow.models.xcom import XCom
from airflow.utils import timezone
-from airflow.utils.state import DagRunState, State, TerminalTIState
+from airflow.utils.state import DagRunState, TaskInstanceState, TerminalTIState
# TODO: Add dependency on JWT token
router = AirflowRouter()
@@ -93,6 +93,9 @@ def ti_run(
TI.try_number,
TI.max_tries,
TI.next_method,
+ TI.hostname,
+ TI.unixname,
+ TI.pid,
# This selects the raw JSON value, by-passing the deserialization
-- we want that to happen on the
# client
column("next_kwargs", JSON),
@@ -102,18 +105,7 @@ def ti_run(
.with_for_update()
)
try:
- (
- previous_state,
- dag_id,
- run_id,
- task_id,
- map_index,
- next_method,
- try_number,
- max_tries,
- next_method,
- next_kwargs,
- ) = session.execute(old).one()
+ ti = session.execute(old).one()
except NoResultFound:
log.error("Task Instance %s not found", ti_id_str)
raise HTTPException(
@@ -129,9 +121,17 @@ def ti_run(
query = update(TI).where(TI.id == ti_id_str).values(data)
- # TODO: We will need to change this for other states like:
- # reschedule, retry, defer etc.
- if previous_state != State.QUEUED:
+ previous_state = ti.state
+
+ # If we are already running, but this is a duplicate request from the same
client return the same OK
+ # -- it's possible there was a network glitch and they never got the
response
+ if previous_state == TaskInstanceState.RUNNING and (ti["hostname"],
ti["unixname"], ti["pid"]) == (
+ ti_run_payload.hostname,
+ ti_run_payload.unixname,
+ ti_run_payload.pid,
+ ):
+ log.info("Duplicate start request received from %s ",
ti_run_payload.hostname)
+ elif previous_state != TaskInstanceState.QUEUED:
log.warning(
"Can not start Task Instance ('%s') in invalid state: %s",
ti_id_str,
@@ -151,14 +151,16 @@ def ti_run(
"previous_state": previous_state,
},
)
- log.info("Task with %s state started on %s ", previous_state,
ti_run_payload.hostname)
+ else:
+ log.info("Task with %s state started on %s ", previous_state,
ti_run_payload.hostname)
# Ensure there is no end date set.
query = query.values(
end_date=None,
hostname=ti_run_payload.hostname,
unixname=ti_run_payload.unixname,
pid=ti_run_payload.pid,
- state=State.RUNNING,
+ state=TaskInstanceState.RUNNING,
+ last_heartbeat_at=timezone.utcnow(),
)
try:
@@ -178,22 +180,21 @@ def ti_run(
DR.run_type,
DR.conf,
DR.logical_date,
- ).filter_by(dag_id=dag_id, run_id=run_id)
+ ).filter_by(dag_id=ti.dag_id, run_id=ti.run_id)
).one_or_none()
if not dr:
- raise ValueError(f"DagRun with dag_id={dag_id} and run_id={run_id}
not found.")
+ raise ValueError(f"DagRun with dag_id={ti.dag_id} and
run_id={ti.run_id} not found.")
# Clear XCom data for the task instance since we are certain it is
executing
# However, do not clear it for deferral
- if not next_method:
- if map_index < 0:
- map_index = None
+ if not ti.next_method:
+ map_index = None if ti.map_index < 0 else ti.map_index
log.info("Clearing xcom data for task id: %s", ti_id_str)
XCom.clear(
- dag_id=dag_id,
- task_id=task_id,
- run_id=run_id,
+ dag_id=ti.dag_id,
+ task_id=ti.task_id,
+ run_id=ti.run_id,
map_index=map_index,
session=session,
)
@@ -203,11 +204,11 @@ def ti_run(
func.count(TaskReschedule.id) # or any other primary key
column
)
.filter(
- TaskReschedule.dag_id == dag_id,
+ TaskReschedule.dag_id == ti.dag_id,
TaskReschedule.task_id == ti_id_str,
- TaskReschedule.run_id == run_id,
+ TaskReschedule.run_id == ti.run_id,
# TaskReschedule.map_index == ti.map_index, # TODO: Handle
mapped tasks
- TaskReschedule.try_number == try_number,
+ TaskReschedule.try_number == ti.try_number,
)
.scalar()
or 0
@@ -216,16 +217,16 @@ def ti_run(
context = TIRunContext(
dag_run=dr,
task_reschedule_count=task_reschedule_count,
- max_tries=max_tries,
+ max_tries=ti.max_tries,
# TODO: Add variables and connections that are needed (and has
perms) for the task
variables=[],
connections=[],
)
# Only set if they are non-null
- if next_method:
- context.next_method = next_method
- context.next_kwargs = next_kwargs
+ if ti.next_method:
+ context.next_method = ti.next_method
+ context.next_kwargs = ti.next_kwargs
return context
except SQLAlchemyError as e:
@@ -291,12 +292,12 @@ def ti_update_state(
# if we get failed, we should attempt to retry, as it is a more
# normal state. Tasks with retries are more frequent than without
retries.
if ti_patch_payload.state == TerminalTIState.FAIL_WITHOUT_RETRY:
- updated_state = State.FAILED
- elif ti_patch_payload.state == State.FAILED:
+ updated_state = TaskInstanceState.FAILED
+ elif ti_patch_payload.state == TaskInstanceState.FAILED:
if _is_eligible_to_retry(previous_state, try_number, max_tries):
- updated_state = State.UP_FOR_RETRY
+ updated_state = TaskInstanceState.UP_FOR_RETRY
else:
- updated_state = State.FAILED
+ updated_state = TaskInstanceState.FAILED
query = query.values(state=updated_state)
elif isinstance(ti_patch_payload, TISuccessStatePayload):
query = TI.duration_expression_update(ti_patch_payload.end_date,
query, session.bind)
@@ -343,13 +344,13 @@ def ti_update_state(
next_kwargs =
BaseSerialization.deserialize(ti_patch_payload.next_kwargs)
query = query.values(
- state=State.DEFERRED,
+ state=TaskInstanceState.DEFERRED,
trigger_id=trigger_row.id,
next_method=ti_patch_payload.next_method,
next_kwargs=next_kwargs,
trigger_timeout=timeout,
)
- updated_state = State.DEFERRED
+ updated_state = TaskInstanceState.DEFERRED
elif isinstance(ti_patch_payload, TIRescheduleStatePayload):
task_instance = session.get(TI, ti_id_str)
actual_start_date = timezone.utcnow()
@@ -370,8 +371,8 @@ def ti_update_state(
# calculate the duration for TI table too
query = TI.duration_expression_update(ti_patch_payload.end_date,
query, session.bind)
# clear the next_method and next_kwargs so that none of the retries
pick them up
- query = query.values(state=State.UP_FOR_RESCHEDULE, next_method=None,
next_kwargs=None)
- updated_state = State.UP_FOR_RESCHEDULE
+ query = query.values(state=TaskInstanceState.UP_FOR_RESCHEDULE,
next_method=None, next_kwargs=None)
+ updated_state = TaskInstanceState.UP_FOR_RESCHEDULE
# TODO: Replace this with FastAPI's Custom Exception handling:
#
https://fastapi.tiangolo.com/tutorial/handling-errors/#install-custom-exception-handlers
try:
@@ -431,7 +432,7 @@ def ti_heartbeat(
},
)
- if previous_state != State.RUNNING:
+ if previous_state != TaskInstanceState.RUNNING:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
@@ -535,7 +536,7 @@ def ti_runtime_checks(
):
ti_id_str = str(task_instance_id)
task_instance = session.scalar(select(TI).where(TI.id == ti_id_str))
- if task_instance.state != State.RUNNING:
+ if task_instance.state != TaskInstanceState.RUNNING:
raise HTTPException(status_code=status.HTTP_409_CONFLICT)
try:
@@ -554,7 +555,7 @@ def ti_runtime_checks(
def _is_eligible_to_retry(state: str, try_number: int, max_tries: int) -> bool:
"""Is task instance is eligible for retry."""
- if state == State.RESTARTING:
+ if state == TaskInstanceState.RESTARTING:
# If a task is cleared when running, it goes into RESTARTING state and
is always
# eligible for retry
return True
diff --git a/tests/api_fastapi/execution_api/routes/test_task_instances.py
b/tests/api_fastapi/execution_api/routes/test_task_instances.py
index 5477869a6cb..684da01e76f 100644
--- a/tests/api_fastapi/execution_api/routes/test_task_instances.py
+++ b/tests/api_fastapi/execution_api/routes/test_task_instances.py
@@ -118,6 +118,37 @@ class TestTIRunState:
assert ti.unixname == "random-unixname"
assert ti.pid == 100
+ response1 = response.json()
+
+ # Test that if we make a second request (simulating a network glitch
so the client issues a retry)
+ # that it is accepted and we get the same info back
+
+ response = client.patch(
+ f"/execution/task-instances/{ti.id}/run",
+ json={
+ "state": "running",
+ "hostname": "random-hostname",
+ "unixname": "random-unixname",
+ "pid": 100,
+ "start_date": instant_str,
+ },
+ )
+ assert response.status_code == 200
+ assert response.json() == response1
+
+ # But that for a different pid on the same host (etc) it fails
+ response = client.patch(
+ f"/execution/task-instances/{ti.id}/run",
+ json={
+ "state": "running",
+ "hostname": "random-hostname",
+ "unixname": "random-unixname",
+ "pid": 101,
+ "start_date": instant_str,
+ },
+ )
+ assert response.status_code == 409
+
def test_next_kwargs_still_encoded(self, client, session,
create_task_instance, time_machine):
instant_str = "2024-09-30T12:00:00Z"
instant = timezone.parse(instant_str)