This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 360a6c4220 Ignore "depends_on_past" for first run in a backfill
(#43131)
360a6c4220 is described below
commit 360a6c4220121956982cf9c4e2265851e85fee39
Author: Daniel Standish <[email protected]>
AuthorDate: Thu Oct 17 16:33:09 2024 -0700
Ignore "depends_on_past" for first run in a backfill (#43131)
Ignore "depends_on_past" for first run in a backfill
This implements this pre-AIP-78 behavior in AIP-78 backfill logic.
Depends on https://github.com/apache/airflow/pull/42684
---
airflow/ti_deps/deps/prev_dagrun_dep.py | 16 ++-
tests/models/test_backfill.py | 113 ++++++++++++++++-
tests/ti_deps/deps/test_prev_dagrun_dep.py | 193 +++++++++++++++++------------
3 files changed, 239 insertions(+), 83 deletions(-)
diff --git a/airflow/ti_deps/deps/prev_dagrun_dep.py
b/airflow/ti_deps/deps/prev_dagrun_dep.py
index 680377de46..7265727f6d 100644
--- a/airflow/ti_deps/deps/prev_dagrun_dep.py
+++ b/airflow/ti_deps/deps/prev_dagrun_dep.py
@@ -21,6 +21,7 @@ from typing import TYPE_CHECKING
from sqlalchemy import func, or_, select
+from airflow.models.backfill import BackfillDagRun
from airflow.models.dagrun import DagRun
from airflow.models.taskinstance import PAST_DEPENDS_MET, TaskInstance as TI
from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
@@ -74,12 +75,13 @@ class PrevDagrunDep(BaseTIDep):
This function exists for easy mocking in tests.
"""
- return exists_query(
+ query = exists_query(
TI.dag_id == ti.dag_id,
TI.task_id == ti.task_id,
TI.execution_date < ti.execution_date,
session=session,
)
+ return query
@staticmethod
def _count_unsuccessful_tis(dagrun: DagRun, task_id: str, *, session:
Session) -> int:
@@ -138,6 +140,17 @@ class PrevDagrunDep(BaseTIDep):
return
dr = ti.get_dagrun(session=session)
+ if dr.backfill_id:
+ sort_ordinal = session.scalar(
+ select(BackfillDagRun.sort_ordinal).where(
+ BackfillDagRun.backfill_id == dr.backfill_id,
+ BackfillDagRun.dag_run_id == dr.id,
+ )
+ )
+ if sort_ordinal == 1:
+ yield self._passing_status(reason="Task instance is first run
in a backfill.")
+ return
+
if not dr:
self._push_past_deps_met_xcom_if_needed(ti, dep_context)
yield self._passing_status(reason="This task instance does not
belong to a DAG.")
@@ -149,7 +162,6 @@ class PrevDagrunDep(BaseTIDep):
last_dagrun = DagRun.get_previous_scheduled_dagrun(dr.id, session)
else:
last_dagrun = DagRun.get_previous_dagrun(dr, session=session)
-
# First ever run for this DAG.
if not last_dagrun:
self._push_past_deps_met_xcom_if_needed(ti, dep_context)
diff --git a/tests/models/test_backfill.py b/tests/models/test_backfill.py
index 50b544a695..3940ac3970 100644
--- a/tests/models/test_backfill.py
+++ b/tests/models/test_backfill.py
@@ -18,12 +18,14 @@
from __future__ import annotations
from contextlib import nullcontext
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING
import pendulum
import pytest
from sqlalchemy import select
-from airflow.models import DagRun
+from airflow.models import DagRun, TaskInstance
from airflow.models.backfill import (
AlreadyRunningBackfill,
Backfill,
@@ -32,7 +34,10 @@ from airflow.models.backfill import (
_create_backfill,
)
from airflow.operators.python import PythonOperator
-from airflow.utils.state import DagRunState
+from airflow.ti_deps.dep_context import DepContext
+from airflow.utils import timezone
+from airflow.utils.state import DagRunState, TaskInstanceState
+from airflow.utils.types import DagRunType
from tests_common.test_utils.db import (
clear_db_backfills,
@@ -41,6 +46,9 @@ from tests_common.test_utils.db import (
clear_db_serialized_dags,
)
+if TYPE_CHECKING:
+ from sqlalchemy.orm import Session
+
pytestmark = [pytest.mark.db_test, pytest.mark.need_serialized_dag]
@@ -175,8 +183,8 @@ def test_cancel_backfill(dag_maker, session):
PythonOperator(task_id="hi", python_callable=print)
b = _create_backfill(
dag_id=dag.dag_id,
- from_date=pendulum.parse("2021-01-01"),
- to_date=pendulum.parse("2021-01-05"),
+ from_date=timezone.datetime(2021, 1, 1),
+ to_date=timezone.datetime(2021, 1, 5),
max_active_runs=2,
reverse=False,
dag_run_conf={},
@@ -199,3 +207,100 @@ def test_cancel_backfill(dag_maker, session):
dag_runs = session.scalars(query).all()
states = [x.state for x in dag_runs]
assert states == ["running", "failed", "failed", "failed", "failed"]
+
+
+def create_next_run(*, is_backfill: bool, next_date: datetime, dag_id: str,
dag_maker, session: Session):
+ """Used in test_ignore_first_depends_on_past to create the next run after
a failed run."""
+ if is_backfill:
+ b = _create_backfill(
+ dag_id=dag_id,
+ from_date=next_date,
+ to_date=next_date + timedelta(days=1),
+ max_active_runs=2,
+ reverse=False,
+ dag_run_conf=None,
+ )
+ assert b
+ # and grab the first dag run from this backfill run
+ next_run = session.scalar(
+ select(DagRun)
+ .join(BackfillDagRun.dag_run)
+ .where(DagRun.backfill_id == b.id)
+ .order_by(BackfillDagRun.sort_ordinal)
+ .limit(1)
+ )
+ return next_run
+ else:
+ dr = dag_maker.create_dagrun(execution_date=next_date,
run_id="second_run")
+ return dr
+
+
[email protected]("is_backfill", [True, False])
[email protected]("catchup", [True, False])
[email protected]("days_between", [1, 10])
[email protected]("first_run_type", [DagRunType.SCHEDULED,
DagRunType.MANUAL])
+def test_ignore_first_depends_on_past(first_run_type, days_between, catchup,
is_backfill, dag_maker, session):
+ """When creating a backfill, should ignore depends_on_past task attr for
the first run in a backfill."""
+ base_date = timezone.datetime(2021, 1, 1)
+ from_date = base_date + timedelta(days=days_between)
+ with dag_maker(dag_id="abc123", serialized=True, catchup=catchup) as dag:
+ op = PythonOperator(task_id="dep_on_past", python_callable=lambda:
print, depends_on_past=True)
+ dr = dag_maker.create_dagrun(execution_date=base_date,
run_type=first_run_type)
+ dr.state = DagRunState.FAILED
+ for ti in dr.task_instances:
+ ti.state = TaskInstanceState.FAILED
+ session.merge(ti)
+ session.commit()
+
+ # let's verify all is as expected
+ session.expunge_all()
+ first_run =
session.scalar(select(DagRun).order_by(DagRun.execution_date).limit(1))
+ assert first_run.state == DagRunState.FAILED
+ tis = first_run.get_task_instances(session=session)
+ assert len(tis) > 0
+ assert all(x.state == TaskInstanceState.FAILED for x in tis)
+
+ next_run = create_next_run(
+ is_backfill=is_backfill,
+ next_date=from_date,
+ dag_id=dag.dag_id,
+ dag_maker=dag_maker,
+ session=session,
+ )
+
+ # check that it's immediately after the other dag run
+ prior_runs = session.scalars(
+ select(DagRun.execution_date).where(DagRun.execution_date <
next_run.execution_date)
+ ).all()
+ assert len(prior_runs) == 1
+ assert prior_runs[0] == first_run.execution_date
+ assert prior_runs[0] + timedelta(days=days_between) ==
next_run.execution_date
+
+ # so now the first backfill dag run follows the other one immediately
+
+ ti: TaskInstance = next_run.get_task_instances(session=session)[0]
+ ti.task = op
+
+ dep_statuses = ti.get_failed_dep_statuses(dep_context=DepContext(),
session=session)
+ if is_backfill:
+ expect_pass = True
+ elif catchup and first_run_type is DagRunType.MANUAL:
+ # this one is a bit weird
+ # if not catchup and first run is manual then it will *not* pass
+ # this is because if it is not catchup it looks at the absolute prior
run
+ # but if it is catchup it looks at only the scheduled prior run
+ # could be a bug ¯\_(ツ)_/¯
+ expect_pass = True
+ else:
+ expect_pass = False
+ if expect_pass:
+ with pytest.raises(StopIteration):
+ next(dep_statuses)
+ assert ti.are_dependencies_met(session=session) is True
+ else:
+ status = next(dep_statuses)
+ assert status.reason == (
+ "depends_on_past is true for this task, "
+ "but 1 previous task instance(s) are not in a successful state."
+ )
+ assert ti.are_dependencies_met(session=session) is False
diff --git a/tests/ti_deps/deps/test_prev_dagrun_dep.py
b/tests/ti_deps/deps/test_prev_dagrun_dep.py
index 0e8b75d2fc..369ddaebbe 100644
--- a/tests/ti_deps/deps/test_prev_dagrun_dep.py
+++ b/tests/ti_deps/deps/test_prev_dagrun_dep.py
@@ -100,135 +100,173 @@ class TestPrevDagrunDep:
@pytest.mark.parametrize(
- (
- "depends_on_past",
- "wait_for_past_depends_before_skipping",
- "wait_for_downstream",
- "prev_tis",
- "context_ignore_depends_on_past",
- "expected_dep_met",
- "past_depends_met_xcom_sent",
- ),
+ "kwargs",
[
# If the task does not set depends_on_past, the previous dagrun should
# be ignored, even though previous_ti would otherwise fail the dep.
# wait_for_past_depends_before_skipping is False, past_depends_met
xcom should not be sent
pytest.param(
- False,
- False,
- False, # wait_for_downstream=True overrides depends_on_past=False.
- [Mock(state=None, **{"are_dependents_done.return_value": False})],
- False,
- True,
- False,
+ dict(
+ depends_on_past=False,
+ wait_for_past_depends_before_skipping=False,
+ wait_for_downstream=False, # wait_for_downstream=True
overrides depends_on_past=False.
+ prev_tis=[Mock(state=None,
**{"are_dependents_done.return_value": False})],
+ context_ignore_depends_on_past=False,
+ expected_dep_met=True,
+ past_depends_met_xcom_sent=False,
+ ),
id="not_depends_on_past",
),
# If the task does not set depends_on_past, the previous dagrun should
# be ignored, even though previous_ti would otherwise fail the dep.
# wait_for_past_depends_before_skipping is True, past_depends_met xcom
should be sent
pytest.param(
- False,
- True,
- False, # wait_for_downstream=True overrides depends_on_past=False.
- [Mock(state=None, **{"are_dependents_done.return_value": False})],
- False,
- True,
- True,
+ dict(
+ depends_on_past=False,
+ wait_for_past_depends_before_skipping=True,
+ wait_for_downstream=False, # wait_for_downstream=True
overrides depends_on_past=False.
+ prev_tis=[Mock(state=None,
**{"are_dependents_done.return_value": False})],
+ context_ignore_depends_on_past=False,
+ expected_dep_met=True,
+ past_depends_met_xcom_sent=True,
+ ),
id="not_depends_on_past",
),
# If the context overrides depends_on_past, the dep should be met even
# though there is no previous_ti which would normally fail the dep.
# wait_for_past_depends_before_skipping is False, past_depends_met
xcom should not be sent
pytest.param(
- True,
- False,
- False,
- [Mock(state=TaskInstanceState.SUCCESS,
**{"are_dependents_done.return_value": True})],
- True,
- True,
- False,
+ dict(
+ depends_on_past=True,
+ wait_for_past_depends_before_skipping=False,
+ wait_for_downstream=False,
+ prev_tis=[
+ Mock(state=TaskInstanceState.SUCCESS,
**{"are_dependents_done.return_value": True})
+ ],
+ context_ignore_depends_on_past=True,
+ expected_dep_met=True,
+ past_depends_met_xcom_sent=False,
+ ),
id="context_ignore_depends_on_past",
),
# If the context overrides depends_on_past, the dep should be met even
# though there is no previous_ti which would normally fail the dep.
# wait_for_past_depends_before_skipping is True, past_depends_met xcom
should be sent
pytest.param(
- True,
- True,
- False,
- [Mock(state=TaskInstanceState.SUCCESS,
**{"are_dependents_done.return_value": True})],
- True,
- True,
- True,
+ dict(
+ depends_on_past=True,
+ wait_for_past_depends_before_skipping=True,
+ wait_for_downstream=False,
+ prev_tis=[
+ Mock(state=TaskInstanceState.SUCCESS,
**{"are_dependents_done.return_value": True})
+ ],
+ context_ignore_depends_on_past=True,
+ expected_dep_met=True,
+ past_depends_met_xcom_sent=True,
+ ),
id="context_ignore_depends_on_past",
),
# The first task run should pass since it has no previous dagrun.
# wait_for_past_depends_before_skipping is False, past_depends_met
xcom should not be sent
- pytest.param(True, False, False, [], False, True, False,
id="first_task_run"),
+ pytest.param(
+ dict(
+ depends_on_past=True,
+ wait_for_past_depends_before_skipping=False,
+ wait_for_downstream=False,
+ prev_tis=[],
+ context_ignore_depends_on_past=False,
+ expected_dep_met=True,
+ past_depends_met_xcom_sent=False,
+ ),
+ id="first_task_run",
+ ),
# The first task run should pass since it has no previous dagrun.
# wait_for_past_depends_before_skipping is True, past_depends_met xcom
should be sent
- pytest.param(True, True, False, [], False, True, True,
id="first_task_run_wait"),
+ pytest.param(
+ dict(
+ depends_on_past=True,
+ wait_for_past_depends_before_skipping=True,
+ wait_for_downstream=False,
+ prev_tis=[],
+ context_ignore_depends_on_past=False,
+ expected_dep_met=True,
+ past_depends_met_xcom_sent=True,
+ ),
+ id="first_task_run_wait",
+ ),
# Previous TI did not complete execution. This dep should fail.
pytest.param(
- True,
- False,
- False,
- [Mock(state=None, **{"are_dependents_done.return_value": True})],
- False,
- False,
- False,
+ dict(
+ depends_on_past=True,
+ wait_for_past_depends_before_skipping=False,
+ wait_for_downstream=False,
+ prev_tis=[Mock(state=None,
**{"are_dependents_done.return_value": True})],
+ context_ignore_depends_on_past=False,
+ expected_dep_met=False,
+ past_depends_met_xcom_sent=False,
+ ),
id="prev_ti_bad_state",
),
# Previous TI specified to wait for the downstream tasks of the
previous
# dagrun. It should fail this dep if the previous TI's downstream TIs
# are not done.
pytest.param(
- True,
- False,
- True,
- [Mock(state=TaskInstanceState.SUCCESS,
**{"are_dependents_done.return_value": False})],
- False,
- False,
- False,
+ dict(
+ depends_on_past=True,
+ wait_for_past_depends_before_skipping=False,
+ wait_for_downstream=True,
+ prev_tis=[
+ Mock(state=TaskInstanceState.SUCCESS,
**{"are_dependents_done.return_value": False})
+ ],
+ context_ignore_depends_on_past=False,
+ expected_dep_met=False,
+ past_depends_met_xcom_sent=False,
+ ),
id="failed_wait_for_downstream",
),
# All the conditions for the dep are met.
# wait_for_past_depends_before_skipping is False, past_depends_met
xcom should not be sent
pytest.param(
- True,
- False,
- True,
- [Mock(state=TaskInstanceState.SUCCESS,
**{"are_dependents_done.return_value": True})],
- False,
- True,
- False,
+ dict(
+ depends_on_past=True,
+ wait_for_past_depends_before_skipping=False,
+ wait_for_downstream=True,
+ prev_tis=[
+ Mock(state=TaskInstanceState.SUCCESS,
**{"are_dependents_done.return_value": True})
+ ],
+ context_ignore_depends_on_past=False,
+ expected_dep_met=True,
+ past_depends_met_xcom_sent=False,
+ ),
id="all_met",
),
# All the conditions for the dep are met
# wait_for_past_depends_before_skipping is False, past_depends_met
xcom should not be sent
pytest.param(
- True,
- True,
- True,
- [Mock(state=TaskInstanceState.SUCCESS,
**{"are_dependents_done.return_value": True})],
- False,
- True,
- True,
+ dict(
+ depends_on_past=True,
+ wait_for_past_depends_before_skipping=True,
+ wait_for_downstream=True,
+ prev_tis=[
+ Mock(state=TaskInstanceState.SUCCESS,
**{"are_dependents_done.return_value": True})
+ ],
+ context_ignore_depends_on_past=False,
+ expected_dep_met=True,
+ past_depends_met_xcom_sent=True,
+ ),
id="all_met",
),
],
)
@patch("airflow.models.dagrun.DagRun.get_previous_scheduled_dagrun")
-def test_dagrun_dep(
- mock_get_previous_scheduled_dagrun,
- depends_on_past,
- wait_for_past_depends_before_skipping,
- wait_for_downstream,
- prev_tis,
- context_ignore_depends_on_past,
- expected_dep_met,
- past_depends_met_xcom_sent,
-):
+def test_dagrun_dep(mock_get_previous_scheduled_dagrun, kwargs):
+ depends_on_past = kwargs["depends_on_past"]
+ wait_for_past_depends_before_skipping =
kwargs["wait_for_past_depends_before_skipping"]
+ wait_for_downstream = kwargs["wait_for_downstream"]
+ prev_tis = kwargs["prev_tis"]
+ context_ignore_depends_on_past = kwargs["context_ignore_depends_on_past"]
+ expected_dep_met = kwargs["expected_dep_met"]
+ past_depends_met_xcom_sent = kwargs["past_depends_met_xcom_sent"]
task = BaseOperator(
task_id="test_task",
dag=DAG("test_dag", schedule=timedelta(days=1),
start_date=datetime(2016, 1, 1)),
@@ -244,6 +282,7 @@ def test_dagrun_dep(
dagrun = Mock(
**{
"get_previous_dagrun.return_value": prev_dagrun,
+ "backfill_id": None,
},
)
ti = Mock(