This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 578ad780389 Fix Task Mapping with XCOM arguments from other Tasks
(#47141)
578ad780389 is described below
commit 578ad780389050d32fd3754c856dcf1e4f48aef2
Author: Amogh Desai <[email protected]>
AuthorDate: Thu Feb 27 21:35:35 2025 +0530
Fix Task Mapping with XCOM arguments from other Tasks (#47141)
---
task_sdk/src/airflow/sdk/api/client.py | 11 ++++++++-
task_sdk/src/airflow/sdk/definitions/xcom_arg.py | 23 +++++++++++++-----
.../src/airflow/sdk/execution_time/supervisor.py | 4 +++-
task_sdk/tests/api/test_client.py | 27 ++++++++++++++++++++++
task_sdk/tests/definitions/test_mappedoperator.py | 10 ++++++--
task_sdk/tests/execution_time/test_supervisor.py | 27 ++++++++++++++++++++++
.../api_fastapi/execution_api/routes/test_xcoms.py | 27 ++++++++++++++++++++++
7 files changed, 119 insertions(+), 10 deletions(-)
diff --git a/task_sdk/src/airflow/sdk/api/client.py
b/task_sdk/src/airflow/sdk/api/client.py
index 46d206ae16e..91bbad42670 100644
--- a/task_sdk/src/airflow/sdk/api/client.py
+++ b/task_sdk/src/airflow/sdk/api/client.py
@@ -298,7 +298,14 @@ class XComOperations:
return XComResponse.model_validate_json(resp.read())
def set(
- self, dag_id: str, run_id: str, task_id: str, key: str, value,
map_index: int | None = None
+ self,
+ dag_id: str,
+ run_id: str,
+ task_id: str,
+ key: str,
+ value,
+ map_index: int | None = None,
+ mapped_length: int | None = None,
) -> dict[str, bool]:
"""Set a XCom value via the API server."""
# TODO: check if we need to use map_index as params in the uri
@@ -306,6 +313,8 @@ class XComOperations:
params = {}
if map_index is not None and map_index >= 0:
params = {"map_index": map_index}
+ if mapped_length is not None and mapped_length >= 0:
+ params["mapped_length"] = mapped_length
self.client.post(f"xcoms/{dag_id}/{run_id}/{task_id}/{key}",
params=params, json=value)
# Any error from the server will anyway be propagated down to the
supervisor,
# so we choose to send a generic response to the supervisor over the
server response to
diff --git a/task_sdk/src/airflow/sdk/definitions/xcom_arg.py
b/task_sdk/src/airflow/sdk/definitions/xcom_arg.py
index 42eef8321f4..a090d0a6f9b 100644
--- a/task_sdk/src/airflow/sdk/definitions/xcom_arg.py
+++ b/task_sdk/src/airflow/sdk/definitions/xcom_arg.py
@@ -338,12 +338,23 @@ class PlainXComArg(XComArg):
if self.operator.is_mapped:
return LazyXComSequence[Any](xcom_arg=self, ti=ti)
-
- result = ti.xcom_pull(
- task_ids=task_id,
- key=self.key,
- default=NOTSET,
- )
+ tg = ti.task.get_closest_mapped_task_group()
+ result = None
+ if tg is None:
+ # regular task
+ result = ti.xcom_pull(
+ task_ids=task_id,
+ key=self.key,
+ default=NOTSET,
+ map_indexes=None,
+ )
+ else:
+ # task from a task group
+ result = ti.xcom_pull(
+ task_ids=task_id,
+ key=self.key,
+ default=NOTSET,
+ )
if not isinstance(result, ArgNotSet):
return result
if self.key == XCOM_RETURN_KEY:
diff --git a/task_sdk/src/airflow/sdk/execution_time/supervisor.py
b/task_sdk/src/airflow/sdk/execution_time/supervisor.py
index 8bc867a3f5b..14c9ee23f50 100644
--- a/task_sdk/src/airflow/sdk/execution_time/supervisor.py
+++ b/task_sdk/src/airflow/sdk/execution_time/supervisor.py
@@ -827,7 +827,9 @@ class ActivitySubprocess(WatchedSubprocess):
self._terminal_state = IntermediateTIState.UP_FOR_RESCHEDULE
self.client.task_instances.reschedule(self.id, msg)
elif isinstance(msg, SetXCom):
- self.client.xcoms.set(msg.dag_id, msg.run_id, msg.task_id,
msg.key, msg.value, msg.map_index)
+ self.client.xcoms.set(
+ msg.dag_id, msg.run_id, msg.task_id, msg.key, msg.value,
msg.map_index, msg.mapped_length
+ )
elif isinstance(msg, PutVariable):
self.client.variables.set(msg.key, msg.value, msg.description)
elif isinstance(msg, SetRenderedFields):
diff --git a/task_sdk/tests/api/test_client.py
b/task_sdk/tests/api/test_client.py
index d2309c3e4ab..abadb7ef683 100644
--- a/task_sdk/tests/api/test_client.py
+++ b/task_sdk/tests/api/test_client.py
@@ -596,6 +596,33 @@ class TestXCOMOperations:
)
assert result == {"ok": True}
+ def test_xcom_set_with_mapped_length(self):
+ # Simulate a successful response from the server when setting an xcom
with mapped_length
+ def handle_request(request: httpx.Request) -> httpx.Response:
+ if (
+ request.url.path == "/xcoms/dag_id/run_id/task_id/key"
+ and request.url.params.get("map_index") == "2"
+ and request.url.params.get("mapped_length") == "3"
+ ):
+ assert json.loads(request.read()) == "value1"
+ return httpx.Response(
+ status_code=201,
+ json={"message": "XCom successfully set"},
+ )
+ return httpx.Response(status_code=400, json={"detail": "Bad
Request"})
+
+ client = make_client(transport=httpx.MockTransport(handle_request))
+ result = client.xcoms.set(
+ dag_id="dag_id",
+ run_id="run_id",
+ task_id="task_id",
+ key="key",
+ value="value1",
+ map_index=2,
+ mapped_length=3,
+ )
+ assert result == {"ok": True}
+
class TestConnectionOperations:
"""
diff --git a/task_sdk/tests/definitions/test_mappedoperator.py
b/task_sdk/tests/definitions/test_mappedoperator.py
index bf03c4e2543..a75cfacf6cb 100644
--- a/task_sdk/tests/definitions/test_mappedoperator.py
+++ b/task_sdk/tests/definitions/test_mappedoperator.py
@@ -570,10 +570,16 @@ def
test_operator_mapped_task_group_receives_value(create_runtime_ti, mock_super
return t2
# The group is mapped by 3.
- t2 = tg.expand(va=[["a", "b"], [4], ["z"]])
+ tg1 = tg.expand(
+ va=[
+ ["a", "b"],
+ [4],
+ ["z"],
+ ]
+ )
# Aggregates results from task group.
- t.override(task_id="t3")(t2)
+ t.override(task_id="t3")(tg1)
def xcom_get():
# TODO: Tidy this after #45927 is reopened and fixed properly
diff --git a/task_sdk/tests/execution_time/test_supervisor.py
b/task_sdk/tests/execution_time/test_supervisor.py
index 114f18c0cf4..f09fe826161 100644
--- a/task_sdk/tests/execution_time/test_supervisor.py
+++ b/task_sdk/tests/execution_time/test_supervisor.py
@@ -1042,6 +1042,7 @@ class TestHandleRequest:
"test_key",
'{"key": "test_key", "value": {"key2": "value2"}}',
None,
+ None,
),
{},
{"ok": True},
@@ -1065,11 +1066,37 @@ class TestHandleRequest:
"test_key",
'{"key": "test_key", "value": {"key2": "value2"}}',
2,
+ None,
),
{},
{"ok": True},
id="set_xcom_with_map_index",
),
+ pytest.param(
+ SetXCom(
+ dag_id="test_dag",
+ run_id="test_run",
+ task_id="test_task",
+ key="test_key",
+ value='{"key": "test_key", "value": {"key2": "value2"}}',
+ map_index=2,
+ mapped_length=3,
+ ),
+ b"",
+ "xcoms.set",
+ (
+ "test_dag",
+ "test_run",
+ "test_task",
+ "test_key",
+ '{"key": "test_key", "value": {"key2": "value2"}}',
+ 2,
+ 3,
+ ),
+ {},
+ {"ok": True},
+ id="set_xcom_with_map_index_and_mapped_length",
+ ),
# we aren't adding all states under TerminalTIState here, because
this test's scope is only to check
# if it can handle TaskState message
pytest.param(
diff --git a/tests/api_fastapi/execution_api/routes/test_xcoms.py
b/tests/api_fastapi/execution_api/routes/test_xcoms.py
index c2de9848726..2f0755fe5a2 100644
--- a/tests/api_fastapi/execution_api/routes/test_xcoms.py
+++ b/tests/api_fastapi/execution_api/routes/test_xcoms.py
@@ -120,6 +120,33 @@ class TestXComsSetEndpoint:
task_map = session.query(TaskMap).filter_by(task_id=ti.task_id,
dag_id=ti.dag_id).one_or_none()
assert task_map is None, "Should not be mapped"
+ def test_xcom_set_mapped(self, client, create_task_instance, session):
+ ti = create_task_instance()
+ session.commit()
+
+ response = client.post(
+ f"/execution/xcoms/{ti.dag_id}/{ti.run_id}/{ti.task_id}/xcom_1",
+ params={"map_index": -1, "mapped_length": 3},
+ json="value1",
+ )
+
+ assert response.status_code == 201
+ assert response.json() == {"message": "XCom successfully set"}
+
+ xcom = (
+ session.query(XCom)
+ .filter_by(task_id=ti.task_id, dag_id=ti.dag_id, key="xcom_1",
map_index=-1)
+ .first()
+ )
+ assert xcom.value == "value1"
+ task_map = session.query(TaskMap).filter_by(task_id=ti.task_id,
dag_id=ti.dag_id).one_or_none()
+ assert task_map is not None, "Should be mapped"
+ assert task_map.dag_id == "dag"
+ assert task_map.run_id == "test"
+ assert task_map.task_id == "op1"
+ assert task_map.map_index == -1
+ assert task_map.length == 3
+
@pytest.mark.parametrize(
("length", "err_context"),
[