This is an automated email from the ASF dual-hosted git repository.

ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new db359ee237 Ensure teardown failure with on_failure_fail_dagrun=True 
fails the DagRun (#30398)
db359ee237 is described below

commit db359ee2375dd7208583aee09b9eae00f1eed1f1
Author: Ephraim Anierobi <[email protected]>
AuthorDate: Mon May 8 11:58:30 2023 +0100

    Ensure teardown failure with on_failure_fail_dagrun=True fails the DagRun 
(#30398)
    
    * Ensure teardown failure with on_failure_fail_dagrun=True fails the DagRun
    
    My approach here is not to include teardown tasks in the leaf_tis when
    deciding on dagrun state except if the teardown task has 
_on_failure_fail_dagrun
    set
    
    * add test for non leave teardown failure
    
    * fix teardown not considered unless they have on_failure_fail_dagrun
    
    * add comment
    
    * Apply suggestions from code review
    
    Co-authored-by: Jed Cunningham 
<[email protected]>
    
    * fixup! Apply suggestions from code review
    
    ---------
    
    Co-authored-by: Jed Cunningham 
<[email protected]>
---
 airflow/models/dag.py       |  11 ++++
 airflow/models/dagrun.py    |  12 ++++
 tests/models/test_dag.py    |  37 +++++++++++-
 tests/models/test_dagrun.py | 142 +++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 200 insertions(+), 2 deletions(-)

diff --git a/airflow/models/dag.py b/airflow/models/dag.py
index d7e6c1230c..9b227db8a5 100644
--- a/airflow/models/dag.py
+++ b/airflow/models/dag.py
@@ -1213,6 +1213,17 @@ class DAG(LoggingMixin):
     def task_ids(self) -> list[str]:
         return list(self.task_dict.keys())
 
+    @property
+    def teardowns(self) -> list[Operator]:
+        return [task for task in self.tasks if getattr(task, "_is_teardown", 
None)]
+
+    @property
+    def tasks_upstream_of_teardowns(self) -> list[Operator]:
+        upstream_tasks = [t.upstream_list for t in self.teardowns]
+        return [
+            val for sublist in upstream_tasks for val in sublist if not 
getattr(val, "_is_teardown", None)
+        ]
+
     @property
     def task_group(self) -> TaskGroup:
         return self._task_group
diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py
index 4ae7e25b27..40ac4ebe10 100644
--- a/airflow/models/dagrun.py
+++ b/airflow/models/dagrun.py
@@ -596,6 +596,18 @@ class DagRun(Base, LoggingMixin):
 
         leaf_task_ids = {t.task_id for t in dag.leaves}
         leaf_tis = [ti for ti in tis if ti.task_id in leaf_task_ids if 
ti.state != TaskInstanceState.REMOVED]
+        # TODO: Remove 'getattr' if setup/teardown is available for mapped task
+        if dag.teardowns:
+            # when on_failure_fail_dagrun is `False`, the final state of the 
DagRun
+            # will be computed as if the teardown task simply didn't exist.
+            teardown_task_ids = [t.task_id for t in dag.teardowns]
+            upstream_of_teardowns = [t.task_id for t in 
dag.tasks_upstream_of_teardowns]
+            teardown_tis = [ti for ti in tis if ti.task_id in 
teardown_task_ids]
+            on_failure_fail_tis = [ti for ti in teardown_tis if 
getattr(ti.task, "_on_failure_fail_dagrun")]
+            tis_upstream_of_teardowns = [ti for ti in tis if ti.task_id in 
upstream_of_teardowns]
+            leaf_tis = list(set(leaf_tis) - set(teardown_tis))
+            leaf_tis.extend(on_failure_fail_tis)
+            leaf_tis.extend(tis_upstream_of_teardowns)
 
         # if all roots finished and at least one failed, the run failed
         if not unfinished.tis and any(leaf_ti.state in State.failed_states for 
leaf_ti in leaf_tis):
diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py
index b0b5b57354..e56f91e71e 100644
--- a/tests/models/test_dag.py
+++ b/tests/models/test_dag.py
@@ -43,7 +43,7 @@ import airflow
 from airflow import models, settings
 from airflow.configuration import conf
 from airflow.datasets import Dataset
-from airflow.decorators import task as task_decorator
+from airflow.decorators import setup, task as task_decorator, teardown
 from airflow.exceptions import (
     AirflowException,
     DuplicateTaskIdFound,
@@ -3205,6 +3205,41 @@ def test_set_task_group_state(run_id, execution_date, 
session, dag_maker):
     }
 
 
+def test_dag_teardowns_property_lists_all_teardown_tasks(dag_maker):
+    @setup
+    def setup_task():
+        return 1
+
+    @teardown
+    def teardown_task():
+        return 1
+
+    @teardown
+    def teardown_task2():
+        return 1
+
+    @teardown
+    def teardown_task3():
+        return 1
+
+    @task_decorator
+    def mytask():
+        return 1
+
+    with dag_maker() as dag:
+        t1 = setup_task()
+        t2 = teardown_task()
+        t3 = teardown_task2()
+        t4 = teardown_task3()
+        with t1 >> t2:
+            with t3:
+                with t4:
+                    mytask()
+
+    assert {t.task_id for t in dag.teardowns} == {"teardown_task", 
"teardown_task2", "teardown_task3"}
+    assert {t.task_id for t in dag.tasks_upstream_of_teardowns} == 
{"setup_task", "mytask"}
+
+
 @pytest.mark.parametrize(
     "start_date, expected_infos",
     [
diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py
index 0756abca35..ffe78b17b6 100644
--- a/tests/models/test_dagrun.py
+++ b/tests/models/test_dagrun.py
@@ -28,7 +28,7 @@ from sqlalchemy.orm.session import Session
 
 from airflow import settings
 from airflow.callbacks.callback_requests import DagCallbackRequest
-from airflow.decorators import task, task_group
+from airflow.decorators import setup, task, task_group, teardown
 from airflow.models import (
     DAG,
     DagBag,
@@ -2313,3 +2313,143 @@ def test_dagrun_with_note(dag_maker, session):
 
     assert session.query(DagRun).filter(DagRun.id == dr.id).one_or_none() is 
None
     assert session.query(DagRunNote).filter(DagRunNote.dag_run_id == 
dr.id).one_or_none() is None
+
+
[email protected](
+    "dag_run_state, on_failure_fail_dagrun", [[DagRunState.SUCCESS, False], 
[DagRunState.FAILED, True]]
+)
+def test_teardown_failure_behaviour_on_dagrun(dag_maker, session, 
dag_run_state, on_failure_fail_dagrun):
+    with dag_maker():
+
+        @teardown(on_failure_fail_dagrun=on_failure_fail_dagrun)
+        def teardowntask():
+            print(1)
+
+        @task
+        def mytask():
+            print(1)
+
+        mytask() >> teardowntask()
+
+    dr = dag_maker.create_dagrun()
+    ti1 = dr.get_task_instance(task_id="mytask")
+    td1 = dr.get_task_instance(task_id="teardowntask")
+    ti1.state = State.SUCCESS
+    td1.state = State.FAILED
+    session.merge(ti1)
+    session.merge(td1)
+    session.flush()
+    dr.update_state()
+    session.flush()
+    dr = session.query(DagRun).one()
+    assert dr.state == dag_run_state
+
+
[email protected](
+    "dag_run_state, on_failure_fail_dagrun", [[DagRunState.SUCCESS, False], 
[DagRunState.FAILED, True]]
+)
+def test_teardown_failure_on_non_leave_behaviour_on_dagrun(
+    dag_maker, session, dag_run_state, on_failure_fail_dagrun
+):
+    with dag_maker():
+
+        @teardown(on_failure_fail_dagrun=on_failure_fail_dagrun)
+        def teardowntask():
+            print(1)
+
+        @teardown
+        def teardowntask2():
+            print(1)
+
+        @task
+        def mytask():
+            print(1)
+
+        mytask() >> teardowntask() >> teardowntask2()
+
+    dr = dag_maker.create_dagrun()
+    ti1 = dr.get_task_instance(task_id="mytask")
+    td1 = dr.get_task_instance(task_id="teardowntask")
+    td2 = dr.get_task_instance(task_id="teardowntask2")
+    ti1.state = State.SUCCESS
+    td1.state = State.FAILED
+    td2.state = State.FAILED
+    session.merge(ti1)
+    session.merge(td1)
+    session.merge(td2)
+    session.flush()
+    dr.update_state()
+    session.flush()
+    dr = session.query(DagRun).one()
+    assert dr.state == dag_run_state
+
+
+def test_work_task_failure_when_setup_teardown_are_successful(dag_maker, 
session):
+    with dag_maker():
+
+        @setup
+        def setuptask():
+            print(2)
+
+        @teardown
+        def teardown_task():
+            print(1)
+
+        @task
+        def mytask():
+            print(1)
+
+        with setuptask() >> teardown_task():
+            mytask()
+
+    dr = dag_maker.create_dagrun()
+    s1 = dr.get_task_instance(task_id="setuptask")
+    td1 = dr.get_task_instance(task_id="teardown_task")
+    t1 = dr.get_task_instance(task_id="mytask")
+    s1.state = TaskInstanceState.SUCCESS
+    td1.state = TaskInstanceState.SUCCESS
+    t1.state = TaskInstanceState.FAILED
+    session.merge(s1)
+    session.merge(td1)
+    session.merge(t1)
+    session.flush()
+    dr.update_state()
+    session.flush()
+    dr = session.query(DagRun).one()
+    assert dr.state == DagRunState.FAILED
+
+
+def test_failure_of_leave_task_not_connected_to_teardown_task(dag_maker, 
session):
+    with dag_maker():
+
+        @setup
+        def setuptask():
+            print(2)
+
+        @teardown
+        def teardown_task():
+            print(1)
+
+        @task
+        def mytask():
+            print(1)
+
+        setuptask()
+        teardown_task()
+        mytask()
+
+    dr = dag_maker.create_dagrun()
+    s1 = dr.get_task_instance(task_id="setuptask")
+    td1 = dr.get_task_instance(task_id="teardown_task")
+    t1 = dr.get_task_instance(task_id="mytask")
+    s1.state = TaskInstanceState.SUCCESS
+    td1.state = TaskInstanceState.SUCCESS
+    t1.state = TaskInstanceState.FAILED
+    session.merge(s1)
+    session.merge(td1)
+    session.merge(t1)
+    session.flush()
+    dr.update_state()
+    session.flush()
+    dr = session.query(DagRun).one()
+    assert dr.state == DagRunState.FAILED

Reply via email to