This is an automated email from the ASF dual-hosted git repository.

jason810496 pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/v3-4-test by this push:
     new 62a0866d5ae [v3-4-test] Add end-to-end tests for task loops (#76886) 
(#77164)
62a0866d5ae is described below

commit 62a0866d5aec5af637578ad4a1669cee0ed5a678
Author: github-actions[bot] 
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Nov 17 04:02:16 2026 +0000

    [v3-4-test] Add end-to-end tests for task loops (#76886) (#77164)
    
    * Add end-to-end tests for task loops
    
    
    
    * Test task loops whose until returns True, and tag the e2e Dags
    
    
    
    ---------
    (cherry picked from commit e306ae6529f2e1e725b69300c2620580ff2d532f)
    
    Signed-off-by: PoAn Yang <[email protected]>
    Co-authored-by: PoAn Yang <[email protected]>
---
 .../basic_tests/test_task_loops.py                 | 312 +++++++++++++++++++++
 .../dags/example_task_loop_test.py                 | 125 +++++++++
 .../airflow_e2e_tests/e2e_test_utils/clients.py    |  14 +
 3 files changed, 451 insertions(+)

diff --git 
a/airflow-e2e-tests/tests/airflow_e2e_tests/basic_tests/test_task_loops.py 
b/airflow-e2e-tests/tests/airflow_e2e_tests/basic_tests/test_task_loops.py
new file mode 100644
index 00000000000..a473488914c
--- /dev/null
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/basic_tests/test_task_loops.py
@@ -0,0 +1,312 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Run task loops in a deployment and check every iteration through the REST 
API."""
+
+from __future__ import annotations
+
+import time
+from collections import Counter
+from datetime import UTC, datetime
+from typing import NamedTuple
+
+import pytest
+
+from airflow_e2e_tests.e2e_test_utils.clients import AirflowClient
+
+
+class ExampleLoop(NamedTuple):
+    iterations: int
+    # This field maps the ID of each task in the loop to that task's map 
indexes.
+    # In the example Dags listed in EXAMPLE_LOOPS, each task's map indexes are 
the same in every iteration.
+    # [-1] stands for a task that is not mapped.
+    loop_tasks: dict[str, list[int]]
+    outside_tasks: list[str]
+
+
+# The keys are the IDs of the example Dags defined in example_task_loops.py.
+EXAMPLE_LOOPS = {
+    "refine_estimate": ExampleLoop(
+        iterations=4,
+        loop_tasks={"refine.improve": [-1], "refine.evaluate": [-1], 
"refine.accurate_enough": [-1]},
+        outside_tasks=["finished"],
+    ),
+    "fixed_task_loop": ExampleLoop(
+        iterations=3,
+        loop_tasks={"accumulate.increment": [-1], "accumulate.__loop_gate": 
[-1]},
+        outside_tasks=[],
+    ),
+    "mapped_task_loop": ExampleLoop(
+        iterations=2,
+        loop_tasks={
+            "process_batch.choose_items": [-1],
+            "process_batch.process": [0, 1],
+            "process_batch.batch_ready": [-1],
+        },
+        outside_tasks=[],
+    ),
+    "partial_override_loop": ExampleLoop(
+        iterations=3,
+        loop_tasks={"accumulate_more.increment": [-1], 
"accumulate_more.__loop_gate": [-1]},
+        outside_tasks=[],
+    ),
+}
+
+LOOP_DAG_ID = "example_task_loop_test"
+ADD_ONE = "accumulate.add_one"
+GATE = "accumulate.__loop_gate"
+MAPPED_LOOP_DAG_ID = "example_task_loop_mapped_test"
+NOT_CONVERGING_DAG_ID = "example_task_loop_not_converging_test"
+UNTIL_MET_EARLY_DAG_ID = "example_task_loop_until_met_early_test"
+UNTIL_MET_AT_LAST_DAG_ID = "example_task_loop_until_met_at_last_test"
+
+# Each key is the name that a test uses to find its Dag run. Each value is the 
ID of the Dag to trigger
+# for that run.
+DAG_RUNS = {
+    **{dag_id: dag_id for dag_id in EXAMPLE_LOOPS},
+    "read_from_outside": LOOP_DAG_ID,
+    "replace_later_iterations": LOOP_DAG_ID,
+    "keep_later_iterations": LOOP_DAG_ID,
+    "mapped": MAPPED_LOOP_DAG_ID,
+    "not_converging": NOT_CONVERGING_DAG_ID,
+    "until_met_early": UNTIL_MET_EARLY_DAG_ID,
+    "until_met_at_last": UNTIL_MET_AT_LAST_DAG_ID,
+}
+
+# Seconds that a Dag run gets to finish. The runs that the dag_runs fixture 
triggers share one deadline. If
+# the loops in all of those runs hang, the tests fail after one timeout in 
total, not one timeout per test.
+RUN_TIMEOUT = 300
+
+# The report task returns this list after the first run of LOOP_DAG_ID. Each 
pair holds the value that
+# add_one returned in one iteration and the trajectory that add_one ran in.
+FIRST_REPORT = [[1, 1], [2, 1], [3, 1]]
+
+# This dict gives the state and try number of each task instance of 
LOOP_DAG_ID after the gate of
+# iteration 2 and the report task are cleared once. A clear that keeps the 
later iterations and a clear that
+# replaces them both lead to these states and try numbers. This holds because 
the replacements of
+# iteration 3 are new task instances, whose try numbers start from 1 again.
+STATES_AFTER_RERUN = {
+    (ADD_ONE, 1, -1): ("success", 1),
+    (GATE, 1, -1): ("success", 1),
+    (ADD_ONE, 2, -1): ("success", 1),
+    (GATE, 2, -1): ("success", 2),
+    (ADD_ONE, 3, -1): ("success", 1),
+    (GATE, 3, -1): ("success", 1),
+    ("report", None, -1): ("success", 2),
+}
+
+airflow_client = AirflowClient()
+
+
+class DagRuns(NamedTuple):
+    run_ids: dict[str, str]
+    deadline: float
+
+
[email protected](scope="module")
+def dag_runs() -> DagRuns:
+    """
+    Trigger one Dag run for each entry of ``DAG_RUNS`` and return the run IDs 
and a deadline.
+
+    The run IDs are keyed like ``DAG_RUNS``, and all the runs share the 
deadline. The fixture triggers all
+    the runs before any test waits for a run, so their loops run at the same 
time.
+    """
+    run_ids = {
+        name: airflow_client.trigger_dag(dag_id, json={"logical_date": 
datetime.now(UTC).isoformat()})[
+            "dag_run_id"
+        ]
+        for name, dag_id in DAG_RUNS.items()
+    }
+    return DagRuns(run_ids, time.monotonic() + RUN_TIMEOUT)
+
+
+def get_task_instances_by_key(dag_id: str, run_id: str) -> dict[tuple[str, int 
| None, int], dict]:
+    """
+    Return the current task instances of a Dag run, keyed by task ID, loop 
iteration and map index.
+
+    The REST API lists only the latest try of each task instance and leaves 
out the iterations that a
+    rerun replaced. The loop iteration is None for a task outside the loop. 
Fail the test if two task
+    instances have the same key.
+    """
+    task_instances = airflow_client.get_task_instances(dag_id, 
run_id)["task_instances"]
+    keys = [
+        (ti["task_id"], ti["loop_iteration"]["iteration"] if ti["in_loop"] 
else None, ti["map_index"])
+        for ti in task_instances
+    ]
+    duplicates = [key for key, count in Counter(keys).items() if count > 1]
+    assert not duplicates, f"More than one current task instance for 
{duplicates}"
+    return dict(zip(keys, task_instances))
+
+
+def get_states_and_tries(task_instances: dict) -> dict:
+    return {key: (ti["state"], ti["try_number"]) for key, ti in 
task_instances.items()}
+
+
+def wait_for_run(dag_id: str, run_id: str, state: str, deadline: float) -> 
dict:
+    """
+    Wait for a Dag run to finish in ``state`` and return what 
``get_task_instances_by_key`` returns for it.
+
+    If the Dag run finishes in another state, or has not finished by 
``deadline``, fail the test. The failure
+    message shows the state and try number of each task instance, so it points 
at the task that went wrong.
+    """
+    try:
+        final_state = airflow_client.wait_for_dag_run(
+            dag_id, run_id, timeout=max(deadline - time.monotonic(), 1)
+        )
+    except TimeoutError:
+        final_state = "unfinished"
+    task_instances = get_task_instances_by_key(dag_id, run_id)
+    assert final_state == state, (
+        f"Dag run {run_id} of {dag_id} is {final_state}, not {state}: 
{get_states_and_tries(task_instances)}"
+    )
+    return task_instances
+
+
+def get_report(dag_id: str, run_id: str):
+    xcom = airflow_client.get_xcom_value(dag_id=dag_id, task_id="report", 
run_id=run_id, key="return_value")
+    return xcom["value"]
+
+
[email protected]("dag_id", EXAMPLE_LOOPS)
+def test_example_loop_runs_every_iteration(dag_id, dag_runs):
+    example = EXAMPLE_LOOPS[dag_id]
+
+    task_instances = wait_for_run(dag_id, dag_runs.run_ids[dag_id], "success", 
dag_runs.deadline)
+
+    expected = {
+        (task_id, iteration, map_index): "success"
+        for task_id, map_indexes in example.loop_tasks.items()
+        for iteration in range(1, example.iterations + 1)
+        for map_index in map_indexes
+    }
+    expected.update({(task_id, None, -1): "success" for task_id in 
example.outside_tasks})
+    assert {key: ti["state"] for key, ti in task_instances.items()} == expected
+
+
+def test_task_outside_loop_reads_every_iteration(dag_runs):
+    run_id = dag_runs.run_ids["read_from_outside"]
+
+    wait_for_run(LOOP_DAG_ID, run_id, "success", dag_runs.deadline)
+
+    assert get_report(LOOP_DAG_ID, run_id) == FIRST_REPORT
+
+
+def test_mapped_task_in_loop_expands_in_every_iteration(dag_runs):
+    run_id = dag_runs.run_ids["mapped"]
+
+    task_instances = wait_for_run(MAPPED_LOOP_DAG_ID, run_id, "success", 
dag_runs.deadline)
+
+    # choose_items returns one more item in each iteration, so the mapped 
`square` task expands to 1, 2 and
+    # then 3 task instances.
+    expected = {
+        ("grow.square", iteration, map_index): "success"
+        for iteration in (1, 2, 3)
+        for map_index in range(iteration)
+    }
+    expected.update(
+        {
+            (task_id, iteration, -1): "success"
+            for task_id in ("grow.choose_items", "grow.__loop_gate")
+            for iteration in (1, 2, 3)
+        }
+    )
+    expected["report", None, -1] = "success"
+    assert {key: ti["state"] for key, ti in task_instances.items()} == expected
+    # `report` gets the squares ordered by iteration first and by map index 
within an iteration.
+    assert get_report(MAPPED_LOOP_DAG_ID, run_id) == [0, 0, 1, 0, 1, 4]
+
+
[email protected](
+    ("run_name", "include_later_loop_iterations", "new_task_instances", 
"new_regions", "report"),
+    [
+        pytest.param(
+            "replace_later_iterations",
+            True,
+            {(GATE, 2, -1), (ADD_ONE, 3, -1), (GATE, 3, -1), ("report", None, 
-1)},
+            # The replacement of iteration 3 runs in a new trajectory of the 
loop, and each trajectory has
+            # its own region. The gate of iteration 2 runs again in the region 
of the first trajectory.
+            {(ADD_ONE, 3, -1), (GATE, 3, -1)},
+            [[1, 1], [2, 1], [3, 2]],
+            id="replace",
+        ),
+        pytest.param(
+            "keep_later_iterations",
+            False,
+            {(GATE, 2, -1), ("report", None, -1)},
+            set(),
+            FIRST_REPORT,
+            id="keep",
+        ),
+    ],
+)
+def test_rerun_from_gate(
+    run_name, include_later_loop_iterations, new_task_instances, new_regions, 
report, dag_runs
+):
+    run_id = dag_runs.run_ids[run_name]
+    before = wait_for_run(LOOP_DAG_ID, run_id, "success", dag_runs.deadline)
+    assert get_states_and_tries(before) == {key: ("success", 1) for key in 
STATES_AFTER_RERUN}
+    assert get_report(LOOP_DAG_ID, run_id) == FIRST_REPORT
+
+    airflow_client.clear_task_instances(
+        LOOP_DAG_ID,
+        run_id,
+        task_instance_ids=[before[GATE, 2, -1]["id"]],
+        include_downstream=True,
+        include_later_loop_iterations=include_later_loop_iterations,
+    )
+
+    after = wait_for_run(LOOP_DAG_ID, run_id, "success", time.monotonic() + 
RUN_TIMEOUT)
+    assert get_states_and_tries(after) == STATES_AFTER_RERUN
+    assert {key for key in after if after[key]["id"] != before[key]["id"]} == 
new_task_instances
+    assert {
+        key for key in after if after[key].get("region_id") != 
before[key].get("region_id")
+    } == new_regions
+    assert get_report(LOOP_DAG_ID, run_id) == report
+
+
+def test_loop_that_never_meets_until_fails_the_run(dag_runs):
+    run_id = dag_runs.run_ids["not_converging"]
+
+    task_instances = wait_for_run(NOT_CONVERGING_DAG_ID, run_id, "failed", 
dag_runs.deadline)
+
+    assert get_states_and_tries(task_instances) == {
+        ("poll.check", 1, -1): ("success", 1),
+        ("poll.reached_ten", 1, -1): ("success", 1),
+        ("poll.check", 2, -1): ("success", 1),
+        # example_task_loop_not_converging_test sets retries=1 in 
default_args. The gate of the last
+        # iteration still fails without retrying, because `until` would not 
return True on a retry either.
+        ("poll.reached_ten", 2, -1): ("failed", 1),
+        ("after", None, -1): ("upstream_failed", 0),
+    }
+
+
[email protected](
+    ("run_name", "dag_id", "iterations"),
+    [
+        pytest.param("until_met_early", UNTIL_MET_EARLY_DAG_ID, 2, id="early"),
+        pytest.param("until_met_at_last", UNTIL_MET_AT_LAST_DAG_ID, 3, 
id="at-last"),
+    ],
+)
+def test_loop_stops_when_until_is_met(run_name, dag_id, iterations, dag_runs):
+    task_instances = wait_for_run(dag_id, dag_runs.run_ids[run_name], 
"success", dag_runs.deadline)
+
+    expected = {
+        (task_id, iteration, -1): ("success", 1)
+        for task_id in ("poll.check", "poll.reached_target")
+        for iteration in range(1, iterations + 1)
+    }
+    expected["after", None, -1] = ("success", 1)
+    assert get_states_and_tries(task_instances) == expected
diff --git 
a/airflow-e2e-tests/tests/airflow_e2e_tests/dags/example_task_loop_test.py 
b/airflow-e2e-tests/tests/airflow_e2e_tests/dags/example_task_loop_test.py
new file mode 100644
index 00000000000..cf3b2779804
--- /dev/null
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/dags/example_task_loop_test.py
@@ -0,0 +1,125 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Dags with task loops for the tests in ``basic_tests/test_task_loops.py``."""
+
+from __future__ import annotations
+
+from datetime import timedelta
+
+from airflow.sdk import dag, task, task_group
+
+
+@dag(schedule=None, catchup=False, tags=["example", "task-loop"])
+def example_task_loop_test():
+    @task_group
+    def accumulate():
+        @task
+        def add_one(*, loop):
+            total = (0 if loop.index == 1 else loop.previous["value"]) + 1
+            return {"value": total, "trajectory": loop.trajectory}
+
+        add_one()
+
+    @task
+    def report(results):
+        return [[result["value"], result["trajectory"]] for result in results]
+
+    accumulation = accumulate.loop(max_iterations=3)
+    report(accumulation["add_one"].output)
+
+
+example_task_loop_test()
+
+
+@dag(schedule=None, catchup=False, tags=["example", "task-loop"])
+def example_task_loop_mapped_test():
+    @task_group
+    def grow():
+        @task
+        def choose_items(*, loop):
+            return list(range(loop.index))
+
+        @task
+        def square(value):
+            return value * value
+
+        square.expand(value=choose_items())
+
+    @task
+    def report(squares):
+        return list(squares)
+
+    growth = grow.loop(max_iterations=3)
+    report(growth["square"].output)
+
+
+example_task_loop_mapped_test()
+
+
+@dag(
+    schedule=None,
+    catchup=False,
+    tags=["example", "task-loop"],
+    default_args={"retries": 1, "retry_delay": timedelta(seconds=1)},
+)
+def example_task_loop_not_converging_test():
+    @task_group
+    def poll():
+        @task
+        def check(*, loop):
+            return loop.index
+
+        check()
+
+    def reached_ten(*, loop):
+        return loop.result >= 10
+
+    @task
+    def after():
+        pass
+
+    poll.loop(max_iterations=2, until=reached_ten) >> after()
+
+
+example_task_loop_not_converging_test()
+
+
+def poll_until(dag_id: str, *, target: int, max_iterations: int):
+    @dag(dag_id=dag_id, schedule=None, catchup=False, tags=["example", 
"task-loop"])
+    def poll_until_target():
+        @task_group
+        def poll():
+            @task
+            def check(*, loop):
+                return loop.index
+
+            check()
+
+        def reached_target(*, loop):
+            return loop.result >= target
+
+        @task
+        def after():
+            pass
+
+        poll.loop(max_iterations=max_iterations, until=reached_target) >> 
after()
+
+    poll_until_target()
+
+
+poll_until("example_task_loop_until_met_early_test", target=2, 
max_iterations=5)
+poll_until("example_task_loop_until_met_at_last_test", target=3, 
max_iterations=3)
diff --git 
a/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py 
b/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
index 821765ad212..ff399a4616f 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
@@ -298,6 +298,20 @@ class AirflowClient:
             params["task_id"] = task_id
         return self._make_request(method="GET", endpoint="eventLogs", 
params=params)["event_logs"]
 
+    def clear_task_instances(self, dag_id: str, run_id: str, **options):
+        """
+        Clear task instances of a Dag run and queue the Dag run again.
+
+        By default the endpoint turns on ``dry_run`` and ``only_failed``. With 
these defaults it only lists
+        the failed task instances that a clear would select. This method turns 
both options off and clears
+        the selected task instances in any state.
+        """
+        return self._make_request(
+            method="POST",
+            endpoint=f"dags/{dag_id}/clearTaskInstances",
+            json={"dag_run_id": run_id, "dry_run": False, "only_failed": 
False, **options},
+        )
+
 
 class TaskSDKClient:
     """Client for interacting with the Task SDK API."""

Reply via email to