This is an automated email from the ASF dual-hosted git repository.

kaxilnaik pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 360a6c4220  Ignore "depends_on_past" for first run in a backfill 
(#43131)
360a6c4220 is described below

commit 360a6c4220121956982cf9c4e2265851e85fee39
Author: Daniel Standish <[email protected]>
AuthorDate: Thu Oct 17 16:33:09 2024 -0700

     Ignore "depends_on_past" for first run in a backfill (#43131)
    
     Ignore "depends_on_past" for first run in a backfill
    
    This implements this pre-AIP-78 behavior in AIP-78 backfill logic.
    
    Depends on https://github.com/apache/airflow/pull/42684
---
 airflow/ti_deps/deps/prev_dagrun_dep.py    |  16 ++-
 tests/models/test_backfill.py              | 113 ++++++++++++++++-
 tests/ti_deps/deps/test_prev_dagrun_dep.py | 193 +++++++++++++++++------------
 3 files changed, 239 insertions(+), 83 deletions(-)

diff --git a/airflow/ti_deps/deps/prev_dagrun_dep.py 
b/airflow/ti_deps/deps/prev_dagrun_dep.py
index 680377de46..7265727f6d 100644
--- a/airflow/ti_deps/deps/prev_dagrun_dep.py
+++ b/airflow/ti_deps/deps/prev_dagrun_dep.py
@@ -21,6 +21,7 @@ from typing import TYPE_CHECKING
 
 from sqlalchemy import func, or_, select
 
+from airflow.models.backfill import BackfillDagRun
 from airflow.models.dagrun import DagRun
 from airflow.models.taskinstance import PAST_DEPENDS_MET, TaskInstance as TI
 from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
@@ -74,12 +75,13 @@ class PrevDagrunDep(BaseTIDep):
 
         This function exists for easy mocking in tests.
         """
-        return exists_query(
+        query = exists_query(
             TI.dag_id == ti.dag_id,
             TI.task_id == ti.task_id,
             TI.execution_date < ti.execution_date,
             session=session,
         )
+        return query
 
     @staticmethod
     def _count_unsuccessful_tis(dagrun: DagRun, task_id: str, *, session: 
Session) -> int:
@@ -138,6 +140,17 @@ class PrevDagrunDep(BaseTIDep):
             return
 
         dr = ti.get_dagrun(session=session)
+        if dr.backfill_id:
+            sort_ordinal = session.scalar(
+                select(BackfillDagRun.sort_ordinal).where(
+                    BackfillDagRun.backfill_id == dr.backfill_id,
+                    BackfillDagRun.dag_run_id == dr.id,
+                )
+            )
+            if sort_ordinal == 1:
+                yield self._passing_status(reason="Task instance is first run 
in a backfill.")
+                return
+
         if not dr:
             self._push_past_deps_met_xcom_if_needed(ti, dep_context)
             yield self._passing_status(reason="This task instance does not 
belong to a DAG.")
@@ -149,7 +162,6 @@ class PrevDagrunDep(BaseTIDep):
             last_dagrun = DagRun.get_previous_scheduled_dagrun(dr.id, session)
         else:
             last_dagrun = DagRun.get_previous_dagrun(dr, session=session)
-
         # First ever run for this DAG.
         if not last_dagrun:
             self._push_past_deps_met_xcom_if_needed(ti, dep_context)
diff --git a/tests/models/test_backfill.py b/tests/models/test_backfill.py
index 50b544a695..3940ac3970 100644
--- a/tests/models/test_backfill.py
+++ b/tests/models/test_backfill.py
@@ -18,12 +18,14 @@
 from __future__ import annotations
 
 from contextlib import nullcontext
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING
 
 import pendulum
 import pytest
 from sqlalchemy import select
 
-from airflow.models import DagRun
+from airflow.models import DagRun, TaskInstance
 from airflow.models.backfill import (
     AlreadyRunningBackfill,
     Backfill,
@@ -32,7 +34,10 @@ from airflow.models.backfill import (
     _create_backfill,
 )
 from airflow.operators.python import PythonOperator
-from airflow.utils.state import DagRunState
+from airflow.ti_deps.dep_context import DepContext
+from airflow.utils import timezone
+from airflow.utils.state import DagRunState, TaskInstanceState
+from airflow.utils.types import DagRunType
 
 from tests_common.test_utils.db import (
     clear_db_backfills,
@@ -41,6 +46,9 @@ from tests_common.test_utils.db import (
     clear_db_serialized_dags,
 )
 
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
 pytestmark = [pytest.mark.db_test, pytest.mark.need_serialized_dag]
 
 
@@ -175,8 +183,8 @@ def test_cancel_backfill(dag_maker, session):
         PythonOperator(task_id="hi", python_callable=print)
     b = _create_backfill(
         dag_id=dag.dag_id,
-        from_date=pendulum.parse("2021-01-01"),
-        to_date=pendulum.parse("2021-01-05"),
+        from_date=timezone.datetime(2021, 1, 1),
+        to_date=timezone.datetime(2021, 1, 5),
         max_active_runs=2,
         reverse=False,
         dag_run_conf={},
@@ -199,3 +207,100 @@ def test_cancel_backfill(dag_maker, session):
     dag_runs = session.scalars(query).all()
     states = [x.state for x in dag_runs]
     assert states == ["running", "failed", "failed", "failed", "failed"]
+
+
+def create_next_run(*, is_backfill: bool, next_date: datetime, dag_id: str, 
dag_maker, session: Session):
+    """Used in test_ignore_first_depends_on_past to create the next run after 
a failed run."""
+    if is_backfill:
+        b = _create_backfill(
+            dag_id=dag_id,
+            from_date=next_date,
+            to_date=next_date + timedelta(days=1),
+            max_active_runs=2,
+            reverse=False,
+            dag_run_conf=None,
+        )
+        assert b
+        # and grab the first dag run from this backfill run
+        next_run = session.scalar(
+            select(DagRun)
+            .join(BackfillDagRun.dag_run)
+            .where(DagRun.backfill_id == b.id)
+            .order_by(BackfillDagRun.sort_ordinal)
+            .limit(1)
+        )
+        return next_run
+    else:
+        dr = dag_maker.create_dagrun(execution_date=next_date, 
run_id="second_run")
+        return dr
+
+
[email protected]("is_backfill", [True, False])
[email protected]("catchup", [True, False])
[email protected]("days_between", [1, 10])
[email protected]("first_run_type", [DagRunType.SCHEDULED, 
DagRunType.MANUAL])
+def test_ignore_first_depends_on_past(first_run_type, days_between, catchup, 
is_backfill, dag_maker, session):
+    """When creating a backfill, should ignore depends_on_past task attr for 
the first run in a backfill."""
+    base_date = timezone.datetime(2021, 1, 1)
+    from_date = base_date + timedelta(days=days_between)
+    with dag_maker(dag_id="abc123", serialized=True, catchup=catchup) as dag:
+        op = PythonOperator(task_id="dep_on_past", python_callable=lambda: 
print, depends_on_past=True)
+    dr = dag_maker.create_dagrun(execution_date=base_date, 
run_type=first_run_type)
+    dr.state = DagRunState.FAILED
+    for ti in dr.task_instances:
+        ti.state = TaskInstanceState.FAILED
+        session.merge(ti)
+    session.commit()
+
+    # let's verify all is as expected
+    session.expunge_all()
+    first_run = 
session.scalar(select(DagRun).order_by(DagRun.execution_date).limit(1))
+    assert first_run.state == DagRunState.FAILED
+    tis = first_run.get_task_instances(session=session)
+    assert len(tis) > 0
+    assert all(x.state == TaskInstanceState.FAILED for x in tis)
+
+    next_run = create_next_run(
+        is_backfill=is_backfill,
+        next_date=from_date,
+        dag_id=dag.dag_id,
+        dag_maker=dag_maker,
+        session=session,
+    )
+
+    # check that it's immediately after the other dag run
+    prior_runs = session.scalars(
+        select(DagRun.execution_date).where(DagRun.execution_date < 
next_run.execution_date)
+    ).all()
+    assert len(prior_runs) == 1
+    assert prior_runs[0] == first_run.execution_date
+    assert prior_runs[0] + timedelta(days=days_between) == 
next_run.execution_date
+
+    # so now the first backfill dag run follows the other one immediately
+
+    ti: TaskInstance = next_run.get_task_instances(session=session)[0]
+    ti.task = op
+
+    dep_statuses = ti.get_failed_dep_statuses(dep_context=DepContext(), 
session=session)
+    if is_backfill:
+        expect_pass = True
+    elif catchup and first_run_type is DagRunType.MANUAL:
+        # this one is a bit weird
+        # if not catchup and first run is manual then it will *not* pass
+        # this is because if it is not catchup it looks at the absolute prior 
run
+        # but if it is catchup it looks at only the scheduled prior run
+        # could be a bug ¯\_(ツ)_/¯
+        expect_pass = True
+    else:
+        expect_pass = False
+    if expect_pass:
+        with pytest.raises(StopIteration):
+            next(dep_statuses)
+        assert ti.are_dependencies_met(session=session) is True
+    else:
+        status = next(dep_statuses)
+        assert status.reason == (
+            "depends_on_past is true for this task, "
+            "but 1 previous task instance(s) are not in a successful state."
+        )
+        assert ti.are_dependencies_met(session=session) is False
diff --git a/tests/ti_deps/deps/test_prev_dagrun_dep.py 
b/tests/ti_deps/deps/test_prev_dagrun_dep.py
index 0e8b75d2fc..369ddaebbe 100644
--- a/tests/ti_deps/deps/test_prev_dagrun_dep.py
+++ b/tests/ti_deps/deps/test_prev_dagrun_dep.py
@@ -100,135 +100,173 @@ class TestPrevDagrunDep:
 
 
 @pytest.mark.parametrize(
-    (
-        "depends_on_past",
-        "wait_for_past_depends_before_skipping",
-        "wait_for_downstream",
-        "prev_tis",
-        "context_ignore_depends_on_past",
-        "expected_dep_met",
-        "past_depends_met_xcom_sent",
-    ),
+    "kwargs",
     [
         # If the task does not set depends_on_past, the previous dagrun should
         # be ignored, even though previous_ti would otherwise fail the dep.
         # wait_for_past_depends_before_skipping is False, past_depends_met 
xcom should not be sent
         pytest.param(
-            False,
-            False,
-            False,  # wait_for_downstream=True overrides depends_on_past=False.
-            [Mock(state=None, **{"are_dependents_done.return_value": False})],
-            False,
-            True,
-            False,
+            dict(
+                depends_on_past=False,
+                wait_for_past_depends_before_skipping=False,
+                wait_for_downstream=False,  # wait_for_downstream=True 
overrides depends_on_past=False.
+                prev_tis=[Mock(state=None, 
**{"are_dependents_done.return_value": False})],
+                context_ignore_depends_on_past=False,
+                expected_dep_met=True,
+                past_depends_met_xcom_sent=False,
+            ),
             id="not_depends_on_past",
         ),
         # If the task does not set depends_on_past, the previous dagrun should
         # be ignored, even though previous_ti would otherwise fail the dep.
         # wait_for_past_depends_before_skipping is True, past_depends_met xcom 
should be sent
         pytest.param(
-            False,
-            True,
-            False,  # wait_for_downstream=True overrides depends_on_past=False.
-            [Mock(state=None, **{"are_dependents_done.return_value": False})],
-            False,
-            True,
-            True,
+            dict(
+                depends_on_past=False,
+                wait_for_past_depends_before_skipping=True,
+                wait_for_downstream=False,  # wait_for_downstream=True 
overrides depends_on_past=False.
+                prev_tis=[Mock(state=None, 
**{"are_dependents_done.return_value": False})],
+                context_ignore_depends_on_past=False,
+                expected_dep_met=True,
+                past_depends_met_xcom_sent=True,
+            ),
             id="not_depends_on_past",
         ),
         # If the context overrides depends_on_past, the dep should be met even
         # though there is no previous_ti which would normally fail the dep.
         # wait_for_past_depends_before_skipping is False, past_depends_met 
xcom should not be sent
         pytest.param(
-            True,
-            False,
-            False,
-            [Mock(state=TaskInstanceState.SUCCESS, 
**{"are_dependents_done.return_value": True})],
-            True,
-            True,
-            False,
+            dict(
+                depends_on_past=True,
+                wait_for_past_depends_before_skipping=False,
+                wait_for_downstream=False,
+                prev_tis=[
+                    Mock(state=TaskInstanceState.SUCCESS, 
**{"are_dependents_done.return_value": True})
+                ],
+                context_ignore_depends_on_past=True,
+                expected_dep_met=True,
+                past_depends_met_xcom_sent=False,
+            ),
             id="context_ignore_depends_on_past",
         ),
         # If the context overrides depends_on_past, the dep should be met even
         # though there is no previous_ti which would normally fail the dep.
         # wait_for_past_depends_before_skipping is True, past_depends_met xcom 
should be sent
         pytest.param(
-            True,
-            True,
-            False,
-            [Mock(state=TaskInstanceState.SUCCESS, 
**{"are_dependents_done.return_value": True})],
-            True,
-            True,
-            True,
+            dict(
+                depends_on_past=True,
+                wait_for_past_depends_before_skipping=True,
+                wait_for_downstream=False,
+                prev_tis=[
+                    Mock(state=TaskInstanceState.SUCCESS, 
**{"are_dependents_done.return_value": True})
+                ],
+                context_ignore_depends_on_past=True,
+                expected_dep_met=True,
+                past_depends_met_xcom_sent=True,
+            ),
             id="context_ignore_depends_on_past",
         ),
         # The first task run should pass since it has no previous dagrun.
         # wait_for_past_depends_before_skipping is False, past_depends_met 
xcom should not be sent
-        pytest.param(True, False, False, [], False, True, False, 
id="first_task_run"),
+        pytest.param(
+            dict(
+                depends_on_past=True,
+                wait_for_past_depends_before_skipping=False,
+                wait_for_downstream=False,
+                prev_tis=[],
+                context_ignore_depends_on_past=False,
+                expected_dep_met=True,
+                past_depends_met_xcom_sent=False,
+            ),
+            id="first_task_run",
+        ),
         # The first task run should pass since it has no previous dagrun.
         # wait_for_past_depends_before_skipping is True, past_depends_met xcom 
should be sent
-        pytest.param(True, True, False, [], False, True, True, 
id="first_task_run_wait"),
+        pytest.param(
+            dict(
+                depends_on_past=True,
+                wait_for_past_depends_before_skipping=True,
+                wait_for_downstream=False,
+                prev_tis=[],
+                context_ignore_depends_on_past=False,
+                expected_dep_met=True,
+                past_depends_met_xcom_sent=True,
+            ),
+            id="first_task_run_wait",
+        ),
         # Previous TI did not complete execution. This dep should fail.
         pytest.param(
-            True,
-            False,
-            False,
-            [Mock(state=None, **{"are_dependents_done.return_value": True})],
-            False,
-            False,
-            False,
+            dict(
+                depends_on_past=True,
+                wait_for_past_depends_before_skipping=False,
+                wait_for_downstream=False,
+                prev_tis=[Mock(state=None, 
**{"are_dependents_done.return_value": True})],
+                context_ignore_depends_on_past=False,
+                expected_dep_met=False,
+                past_depends_met_xcom_sent=False,
+            ),
             id="prev_ti_bad_state",
         ),
         # Previous TI specified to wait for the downstream tasks of the 
previous
         # dagrun. It should fail this dep if the previous TI's downstream TIs
         # are not done.
         pytest.param(
-            True,
-            False,
-            True,
-            [Mock(state=TaskInstanceState.SUCCESS, 
**{"are_dependents_done.return_value": False})],
-            False,
-            False,
-            False,
+            dict(
+                depends_on_past=True,
+                wait_for_past_depends_before_skipping=False,
+                wait_for_downstream=True,
+                prev_tis=[
+                    Mock(state=TaskInstanceState.SUCCESS, 
**{"are_dependents_done.return_value": False})
+                ],
+                context_ignore_depends_on_past=False,
+                expected_dep_met=False,
+                past_depends_met_xcom_sent=False,
+            ),
             id="failed_wait_for_downstream",
         ),
         # All the conditions for the dep are met.
         # wait_for_past_depends_before_skipping is False, past_depends_met 
xcom should not be sent
         pytest.param(
-            True,
-            False,
-            True,
-            [Mock(state=TaskInstanceState.SUCCESS, 
**{"are_dependents_done.return_value": True})],
-            False,
-            True,
-            False,
+            dict(
+                depends_on_past=True,
+                wait_for_past_depends_before_skipping=False,
+                wait_for_downstream=True,
+                prev_tis=[
+                    Mock(state=TaskInstanceState.SUCCESS, 
**{"are_dependents_done.return_value": True})
+                ],
+                context_ignore_depends_on_past=False,
+                expected_dep_met=True,
+                past_depends_met_xcom_sent=False,
+            ),
             id="all_met",
         ),
         # All the conditions for the dep are met
         # wait_for_past_depends_before_skipping is False, past_depends_met 
xcom should not be sent
         pytest.param(
-            True,
-            True,
-            True,
-            [Mock(state=TaskInstanceState.SUCCESS, 
**{"are_dependents_done.return_value": True})],
-            False,
-            True,
-            True,
+            dict(
+                depends_on_past=True,
+                wait_for_past_depends_before_skipping=True,
+                wait_for_downstream=True,
+                prev_tis=[
+                    Mock(state=TaskInstanceState.SUCCESS, 
**{"are_dependents_done.return_value": True})
+                ],
+                context_ignore_depends_on_past=False,
+                expected_dep_met=True,
+                past_depends_met_xcom_sent=True,
+            ),
             id="all_met",
         ),
     ],
 )
 @patch("airflow.models.dagrun.DagRun.get_previous_scheduled_dagrun")
-def test_dagrun_dep(
-    mock_get_previous_scheduled_dagrun,
-    depends_on_past,
-    wait_for_past_depends_before_skipping,
-    wait_for_downstream,
-    prev_tis,
-    context_ignore_depends_on_past,
-    expected_dep_met,
-    past_depends_met_xcom_sent,
-):
+def test_dagrun_dep(mock_get_previous_scheduled_dagrun, kwargs):
+    depends_on_past = kwargs["depends_on_past"]
+    wait_for_past_depends_before_skipping = 
kwargs["wait_for_past_depends_before_skipping"]
+    wait_for_downstream = kwargs["wait_for_downstream"]
+    prev_tis = kwargs["prev_tis"]
+    context_ignore_depends_on_past = kwargs["context_ignore_depends_on_past"]
+    expected_dep_met = kwargs["expected_dep_met"]
+    past_depends_met_xcom_sent = kwargs["past_depends_met_xcom_sent"]
     task = BaseOperator(
         task_id="test_task",
         dag=DAG("test_dag", schedule=timedelta(days=1), 
start_date=datetime(2016, 1, 1)),
@@ -244,6 +282,7 @@ def test_dagrun_dep(
     dagrun = Mock(
         **{
             "get_previous_dagrun.return_value": prev_dagrun,
+            "backfill_id": None,
         },
     )
     ti = Mock(

Reply via email to