This is an automated email from the ASF dual-hosted git repository.
ephraimbuddy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 5484ce8301a Fix DagRun.dag_versions after partial
run_on_latest_version clear (#71459)
5484ce8301a is described below
commit 5484ce8301a163f12e1fbed58f61dd019fddef84
Author: Digvijay <[email protected]>
AuthorDate: Wed Nov 18 01:46:53 2026 -0600
Fix DagRun.dag_versions after partial run_on_latest_version clear (#71459)
* Fix DagRun.dag_versions after partial run_on_latest_version clear
Stop special-casing bundle-versioned runs through created_dag_version_id.
Always derive versions from the run's TaskInstance / history rows so mixed
versions after a partial clear are reported correctly.
Closes #71454
Signed-off-by: Digvijay <[email protected]>
* Prefetch dag versions for bundle-versioned DagRuns
attach_dag_versions_to_runs previously skipped runs with bundle_version,
assuming dag_versions used created_dag_version. After removing that
shortcut, prefetch all runs so Grid/UI keeps the lightweight path and
still sees mixed TI versions.
Signed-off-by: Digvijay <[email protected]>
* Address review nits on bundled dag_versions tests
Shorten the dag_versions comment, drop the issue number from the
test, and name the case for bundled DAGs.
Signed-off-by: Digvijay <[email protected]>
* Add full run_on_latest_version clear coverage for dag_versions
A full clear moves every current TI to latest, so the older version
only remains on TIH. Cover that path and keep prefetch from skipping
bundle-versioned runs.
Signed-off-by: Digvijay <[email protected]>
* Prefetch dag versions on single DagRun GET
get_dag_run() skipped attach_dag_versions_to_runs, so opening a large
mapped run materialized every TI/TIH row just to return versions.
Signed-off-by: Digvijay <[email protected]>
* Assert get_dag_run leaves TI collections unloaded
Query count stayed flat even when each query loaded every TI/TIH row.
Serialize a bundle-versioned run with history and assert those
collections stay unloaded.
Signed-off-by: Digvijay <[email protected]>
* Prefetch dag versions on DagRun PATCH
A note-only update still serializes dag_versions, so without the
DISTINCT helper it walks every TI/TIH row. Reload tests by primary
key because run_id is only unique within a Dag.
Signed-off-by: Digvijay <[email protected]>
* Move DISTINCT dag-version lookup into DagRun.dag_versions
Session-attached runs now use one DISTINCT query over current and archived
attempts, so clear, trigger, and other single-run routes serialize versions
without walking TI collections. include_all_attempts=True is required after
attempt archiving so a full run_on_latest_version clear still surfaces the
older version.
Signed-off-by: Digvijay <[email protected]>
* Tighten dag_versions types and serialization tests
Annotate fetch_dag_versions_for_runs, give the transient DagVersion a
real id, and expire TI collections before asserting serialization so
the check measures the DISTINCT fallback rather than leftover identity-map
state from clear.
Signed-off-by: Digvijay <[email protected]>
* Address review on dag_versions lookup and route tests
Collapse the version fetch into one query, restore the lighter GET load,
and keep only the response checks that still cover archived attempts.
Signed-off-by: Digvijay <[email protected]>
* Prefetch versions on bulk clear and fall back when attempts have none
Bulk clear serializes dag_versions, so attach the batch prefetch before
building the response. Runs with no attempt version report
created_dag_version.
Signed-off-by: Digvijay <[email protected]>
* Drop created_dag_version fallback when attempts have no version
Runs from before Dag versioning have a null created_dag_version_id, and
runs created since then already carry a version on their task instances.
Returning created_dag_version for an empty attempt set also changed
unpinned runs that previously returned an empty list.
Signed-off-by: Digvijay <[email protected]>
---------
Signed-off-by: Digvijay <[email protected]>
---
.../src/airflow/api_fastapi/common/db/dag_runs.py | 64 ++---------
.../api_fastapi/core_api/routes/public/dag_run.py | 1 +
airflow-core/src/airflow/models/dagrun.py | 55 ++++++---
.../core_api/routes/public/test_dag_run.py | 124 ++++++++++++++++++++-
airflow-core/tests/unit/models/test_dagrun.py | 121 +++++++++++++++++++-
5 files changed, 289 insertions(+), 76 deletions(-)
diff --git a/airflow-core/src/airflow/api_fastapi/common/db/dag_runs.py
b/airflow-core/src/airflow/api_fastapi/common/db/dag_runs.py
index 71168ebc3e6..0e12c94cf54 100644
--- a/airflow-core/src/airflow/api_fastapi/common/db/dag_runs.py
+++ b/airflow-core/src/airflow/api_fastapi/common/db/dag_runs.py
@@ -17,12 +17,10 @@
from __future__ import annotations
-from collections import defaultdict
from collections.abc import Sequence
from typing import TYPE_CHECKING
-from uuid import UUID
-from sqlalchemy import func, select, tuple_
+from sqlalchemy import func, select
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.interfaces import LoaderOption
@@ -30,7 +28,6 @@ from airflow.api_fastapi.common.db.dags import
eager_load_teams
from airflow.models.dag import DagModel
from airflow.models.dag_version import DagVersion
from airflow.models.dagrun import DagRun
-from airflow.models.taskinstance import TaskInstance
if TYPE_CHECKING:
from sqlalchemy.orm import Session
@@ -70,62 +67,17 @@ def eager_load_dag_run_for_list() -> tuple[LoaderOption,
...]:
def attach_dag_versions_to_runs(dag_runs: Sequence[DagRun], *, session:
Session) -> None:
"""
- Prefetch distinct dag_version_ids for each DagRun via a lightweight query.
+ Batch-prefetch dag versions for many runs in one query (list/grid
endpoints).
- Instead of loading all attempt rows (potentially thousands per run)
- through the ORM relationship just to extract distinct dag_version_ids,
- this issues a single query that returns only the distinct
- (dag_id, run_id, dag_version_id) tuples for the given runs.
-
- The result is attached to each DagRun as ``_prefetched_dag_version_ids``
- (a dict mapping version_id -> DagVersion), which the ``dag_versions``
- property reads as an optimized substitute for traversing attempt
- relationships. All business logic (bundle_version shortcut, sorting,
- deduplication) remains solely in ``DagRun.dag_versions``.
+ Bundle-versioned runs are included: after a ``run_on_latest_version``
+ clear they can keep older versions on uncleared TIs or archived attempts,
+ so they must not skip this prefetch.
"""
if not dag_runs:
return
-
- # Only runs without a bundle_version need attempt traversal;
- # runs with bundle_version use created_dag_version directly
- # (handled by the dag_versions property).
- runs_needing_versions = [dr for dr in dag_runs if not dr.bundle_version]
- if not runs_needing_versions:
- return
-
- run_key_values = [(dr.dag_id, dr.run_id) for dr in runs_needing_versions]
-
- rows = session.execute(
- select(
- TaskInstance.dag_id,
- TaskInstance.run_id,
- TaskInstance.dag_version_id.label("dag_version_id"),
- )
- .where(
- TaskInstance.dag_version_id.is_not(None),
- tuple_(TaskInstance.dag_id,
TaskInstance.run_id).in_(run_key_values),
- )
- .distinct()
- ).all()
-
- all_version_ids = {r.dag_version_id for r in rows}
- versions_by_id: dict[UUID, DagVersion] = {}
- if all_version_ids:
- dv_query = (
- select(DagVersion)
- .where(DagVersion.id.in_(all_version_ids))
- .options(joinedload(DagVersion.bundle))
- )
- versions_by_id = {dv.id: dv for dv in session.scalars(dv_query)}
-
- versions_per_run: dict[tuple[str, str], dict[UUID, DagVersion]] =
defaultdict(dict)
- for row in rows:
- dv = versions_by_id.get(row.dag_version_id)
- if dv:
- versions_per_run[(row.dag_id, row.run_id)][dv.id] = dv
-
- for dr in runs_needing_versions:
- dr._prefetched_dag_version_ids = versions_per_run.get((dr.dag_id,
dr.run_id), {})
+ per_run = DagRun.fetch_dag_versions_for_runs([(dr.dag_id, dr.run_id) for
dr in dag_runs], session=session)
+ for dr in dag_runs:
+ dr._prefetched_dag_version_ids = per_run.get((dr.dag_id, dr.run_id),
{})
def is_filtered_existing_run(dag_id: str, run_id: str, *, has_filter: bool,
session: Session) -> bool:
diff --git
a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py
b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py
index 1728013aac6..50098b26ed9 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py
@@ -471,6 +471,7 @@ def clear_dag_runs(
user=user,
)
)
+ attach_dag_versions_to_runs(cleared_runs, session=session)
return DAGRunCollectionResponse(
dag_runs=[DAGRunResponse.model_validate(cleared_run) for cleared_run
in cleared_runs],
total_entries=len(cleared_runs),
diff --git a/airflow-core/src/airflow/models/dagrun.py
b/airflow-core/src/airflow/models/dagrun.py
index 13e03a11654..a000a5d51da 100644
--- a/airflow-core/src/airflow/models/dagrun.py
+++ b/airflow-core/src/airflow/models/dagrun.py
@@ -22,7 +22,7 @@ import logging
import os
import re
from collections import defaultdict
-from collections.abc import Callable, Iterable, Iterator, Sequence
+from collections.abc import Callable, Collection, Iterable, Iterator, Sequence
from dataclasses import dataclass, field, fields, replace
from datetime import datetime
from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar, cast, overload
@@ -65,6 +65,7 @@ from sqlalchemy.orm import (
joinedload,
lazyload,
mapped_column,
+ object_session,
relationship,
synonym,
validates,
@@ -600,22 +601,50 @@ class DagRun(Base, LoggingMixin):
partition_date=self.partition_date,
)
+ @classmethod
+ def fetch_dag_versions_for_runs(
+ cls, run_keys: Collection[tuple[str, str]], *, session: Session
+ ) -> dict[tuple[str, str], dict[UUID, DagVersion]]:
+ """Distinct DagVersions per (dag_id, run_id) across current and
archived attempts, one query."""
+ from airflow.models.dag_version import DagVersion
+
+ if not run_keys:
+ return {}
+ # Distinct on the id only. DagVersion.version_data is JSON, which
PostgreSQL cannot DISTINCT.
+ distinct_versions = (
+ select(TI.dag_id, TI.run_id, TI.dag_version_id)
+ .where(TI.dag_version_id.is_not(None), tuple_(TI.dag_id,
TI.run_id).in_(list(run_keys)))
+ .distinct()
+ .subquery()
+ )
+ rows = session.execute(
+ select(distinct_versions.c.dag_id, distinct_versions.c.run_id,
DagVersion)
+ .join(DagVersion, DagVersion.id ==
distinct_versions.c.dag_version_id)
+ .options(joinedload(DagVersion.bundle))
+ .execution_options(include_all_attempts=True)
+ ).all()
+ out: dict[tuple[str, str], dict[UUID, DagVersion]] = defaultdict(dict)
+ for dag_id, run_id, dv in rows:
+ out[(dag_id, run_id)][dv.id] = dv
+ return out
+
@property
def dag_versions(self) -> list[DagVersion]:
- """Return the DAG versions associated with the TIs of this DagRun."""
- # when the dag is in a versioned bundle, we keep the dag version fixed
- if self.bundle_version:
- return [self.created_dag_version] if self.created_dag_version is
not None else []
-
+ """Return the Dag versions used by any attempt of this DagRun, current
or archived."""
+ versions: Iterable[DagVersion | None]
if self._prefetched_dag_version_ids is not None:
- dag_version_objects =
list(self._prefetched_dag_version_ids.values())
+ versions = self._prefetched_dag_version_ids.values()
+ elif (session := object_session(self)) is not None:
+ # One DISTINCT query; never loads task_instances /
historical_task_instances.
+ versions = (
+ self.fetch_dag_versions_for_runs([(self.dag_id, self.run_id)],
session=session)
+ .get((self.dag_id, self.run_id), {})
+ .values()
+ )
else:
- dag_version_objects = [
- dv
- for dv in dict.fromkeys(list(self._historical_dag_versions) +
list(self._ti_dag_versions))
- if dv is not None
- ]
- return sorted(dag_version_objects, key=lambda dv: dv.id)
+ # Transient run (e.g. built in memory in tests): use the in-memory
relationships.
+ versions = dict.fromkeys(list(self._historical_dag_versions) +
list(self._ti_dag_versions))
+ return sorted((dv for dv in versions if dv is not None), key=lambda
dv: dv.id)
@property
def version_number(self) -> int | None:
diff --git
a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py
b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py
index 3cf4eaeb009..c2632403cda 100644
--- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py
+++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py
@@ -36,15 +36,24 @@ from airflow._shared.timezones import timezone
from airflow.api_fastapi.auth.managers.models.resource_details import
DagAccessEntity, DagDetails
from airflow.api_fastapi.auth.managers.simple.user import SimpleAuthManagerUser
from airflow.api_fastapi.common.dagbag import resolve_run_on_latest_version
+from airflow.api_fastapi.common.db.dag_runs import attach_dag_versions_to_runs
+from airflow.api_fastapi.core_api.datamodels.dag_run import (
+ BulkDAGRunBody,
+ BulkDAGRunClearBody,
+)
from airflow.api_fastapi.core_api.datamodels.dag_versions import
DagVersionResponse
+from airflow.api_fastapi.core_api.routes.public.dag_run import clear_dag_runs
from airflow.api_fastapi.core_api.services.public.dag_run import DagRunWaiter
from airflow.exceptions import ParamValidationError
from airflow.models import DagModel, DagRun, DagTag, Log
from airflow.models.asset import AssetEvent, AssetModel
+from airflow.models.dag_version import DagVersion
+from airflow.models.dagbag import DBDagBag
from airflow.models.dagbundle import DagBundleModel
from airflow.models.dynamic_region import DynamicRegion
+from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.task_state_store import TaskStateStoreModel
-from airflow.models.taskinstance import TaskInstance
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
from airflow.models.team import Team
from airflow.models.xcom import XComModel
from airflow.providers.standard.operators.empty import EmptyOperator
@@ -75,7 +84,6 @@ from tests_common.test_utils.team import attach_dag_to_team
from unit.listeners.class_listener import ClassBasedListener
if TYPE_CHECKING:
- from airflow.models.dag_version import DagVersion
from airflow.timetables.base import DataInterval
pytestmark = pytest.mark.db_test
@@ -147,6 +155,65 @@ DAG2_PARAM = {"validated_number": Param(1, minimum=1,
maximum=10)}
DAG_RUNS_LIST = [DAG1_RUN1_ID, DAG1_RUN2_ID, DAG2_RUN1_ID, DAG2_RUN2_ID]
+def _create_bundled_run_with_history(dag_id, dag_maker, session):
+ with dag_maker(
+ dag_id=dag_id, schedule=None, start_date=START_DATE1,
bundle_version="v1", serialized=True
+ ):
+ EmptyOperator(task_id="0")
+ EmptyOperator(task_id="1")
+ dag_run = dag_maker.create_dagrun(state=DagRunState.SUCCESS)
+ old_dag_version = DagVersion.get_latest_version(dag_id)
+ for ti in dag_run.task_instances:
+ ti.state = TaskInstanceState.SUCCESS
+ session.merge(ti)
+ session.flush()
+
+ with dag_maker(
+ dag_id=dag_id, schedule=None, start_date=START_DATE1,
bundle_version="v2", serialized=True
+ ):
+ EmptyOperator(task_id="0")
+ EmptyOperator(task_id="1")
+ new_dag_version = DagVersion.get_latest_version(dag_id)
+ assert old_dag_version.id != new_dag_version.id
+
+ clear_task_instances(list(dag_run.task_instances), session,
run_on_latest_version=True)
+ session.commit()
+
+ archived_version_ids = set(
+ session.scalars(
+ select(TaskInstance.dag_version_id)
+ .where(
+ TaskInstance.dag_id == dag_id,
+ TaskInstance.run_id == dag_run.run_id,
+ TaskInstance.dag_version_id.is_not(None),
+ )
+ .execution_options(include_all_attempts=True)
+ )
+ )
+ assert old_dag_version.id in archived_version_ids
+ return dag_run.run_id, old_dag_version, new_dag_version
+
+
+def _dag_bag_from_serialized(session):
+ """DagBag that resolves DAGs from SerializedDagModel in the current
session."""
+ dag_bag = mock.Mock(spec=DBDagBag)
+
+ def get_dag_for_run(dag_run, session=session):
+ sdm = None
+ if dag_run.created_dag_version_id is not None:
+ sdm = session.scalar(
+ select(SerializedDagModel).where(
+ SerializedDagModel.dag_version_id ==
dag_run.created_dag_version_id
+ )
+ )
+ if sdm is None:
+ sdm = SerializedDagModel.get(dag_run.dag_id, session=session)
+ return None if sdm is None else sdm.dag
+
+ dag_bag.get_dag_for_run.side_effect = get_dag_for_run
+ return dag_bag
+
+
@pytest.fixture(autouse=True)
@provide_session
def setup(request, dag_maker, *, session=None):
@@ -419,6 +486,20 @@ class TestGetDagRun:
body = response.json()
assert body["detail"] == "The DagRun with dag_id: `test_dag1` and
run_id: `invalid` was not found"
+ @pytest.mark.usefixtures("configure_git_connection_for_dag_bundle")
+ def test_get_dag_run_includes_archived_attempt_versions(self, test_client,
dag_maker, session):
+ dag_id = "test_get_dag_run_prefetch_history"
+ run_id, old_dag_version, new_dag_version =
_create_bundled_run_with_history(
+ dag_id, dag_maker, session
+ )
+
+ response = test_client.get(f"/dags/{dag_id}/dagRuns/{run_id}")
+ assert response.status_code == 200
+ assert {dv["id"] for dv in response.json()["dag_versions"]} == {
+ str(old_dag_version.id),
+ str(new_dag_version.id),
+ }
+
def test_should_respond_401(self, unauthenticated_test_client):
response =
unauthenticated_test_client.get(f"/dags/{DAG1_ID}/dagRuns/invalid")
assert response.status_code == 401
@@ -2616,6 +2697,45 @@ class TestClearDagRun:
class TestBulkClearDagRuns:
+ @pytest.mark.usefixtures("configure_git_connection_for_dag_bundle")
+ @mock.patch(
+
"airflow.api_fastapi.core_api.routes.public.dag_run.attach_dag_versions_to_runs",
+ autospec=True,
+ wraps=attach_dag_versions_to_runs,
+ )
+ def test_clear_dag_runs_includes_archived_attempt_versions(self,
mock_attach, dag_maker, session):
+ run_id_a, old_a, new_a = _create_bundled_run_with_history(
+ "test_bulk_clear_prefetch_history_a", dag_maker, session
+ )
+ run_id_b, old_b, new_b = _create_bundled_run_with_history(
+ "test_bulk_clear_prefetch_history_b", dag_maker, session
+ )
+
+ session.expire_all()
+ result = clear_dag_runs(
+ dag_id="~",
+ body=BulkDAGRunClearBody(
+ dry_run=False,
+ dag_runs=[
+
BulkDAGRunBody(dag_id="test_bulk_clear_prefetch_history_a",
dag_run_id=run_id_a),
+
BulkDAGRunBody(dag_id="test_bulk_clear_prefetch_history_b",
dag_run_id=run_id_b),
+ ],
+ ),
+ dag_bag=_dag_bag_from_serialized(session),
+ session=session,
+ user=SimpleAuthManagerUser(username="test", role="admin"),
+ )
+ assert result.total_entries == 2
+ versions_by_key = {
+ ("test_bulk_clear_prefetch_history_a", run_id_a): (old_a, new_a),
+ ("test_bulk_clear_prefetch_history_b", run_id_b): (old_b, new_b),
+ }
+ for resp in result.dag_runs:
+ old_dag_version, new_dag_version = versions_by_key[(resp.dag_id,
resp.dag_run_id)]
+ assert {dv.id for dv in resp.dag_versions} == {old_dag_version.id,
new_dag_version.id}
+ mock_attach.assert_called_once()
+ assert len(mock_attach.call_args.args[0]) == 2
+
@pytest.mark.usefixtures("configure_git_connection_for_dag_bundle")
def test_bulk_clear_specific_dag(self, test_client, session):
"""Specific dag_id in URL, dag_run_id in body — clears both runs and
queues them."""
diff --git a/airflow-core/tests/unit/models/test_dagrun.py
b/airflow-core/tests/unit/models/test_dagrun.py
index b6888ee5681..e5ab3320386 100644
--- a/airflow-core/tests/unit/models/test_dagrun.py
+++ b/airflow-core/tests/unit/models/test_dagrun.py
@@ -32,6 +32,7 @@ from uuid import UUID
import pendulum
import pytest
import time_machine
+import uuid6
from opentelemetry import trace as otel_trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
@@ -55,6 +56,7 @@ from airflow._shared.observability.traces import (
OverrideableRandomIdGenerator,
)
from airflow._shared.timezones import timezone
+from airflow.api_fastapi.common.db.dag_runs import attach_dag_versions_to_runs
from airflow.api_fastapi.execution_api.datamodels.taskinstance import
TaskInstance as TIDataModel
from airflow.callbacks.callback_requests import DagCallbackRequest,
DagRunContext
from airflow.dag_processing.dagbag import DagBag
@@ -1823,7 +1825,7 @@ class TestDagRun:
assert dag_run.version_number == dag_v.version_number
def test_dag_run_dag_versions_with_null_created_dag_version(self,
dag_maker, session):
- """Test that dag_versions returns empty list when created_dag_version
is None and bundle_version is populated."""
+ """bundle_version alone must not hide TI versions when
created_dag_version is None."""
with dag_maker(
"test_dag_run_null_created_dag_version",
schedule=datetime.timedelta(days=1),
@@ -1832,16 +1834,125 @@ class TestDagRun:
EmptyOperator(task_id="empty")
dag_run = dag_maker.create_dagrun()
+ ti_version_ids = {ti.dag_version_id for ti in dag_run.task_instances
if ti.dag_version_id is not None}
+ assert ti_version_ids
+
dag_run.bundle_version = "some_bundle_version"
dag_run.created_dag_version_id = None
dag_run.created_dag_version = None
session.merge(dag_run)
session.flush()
- # This should return empty list, not [None]
- assert dag_run.dag_versions == []
- assert isinstance(dag_run.dag_versions, list)
- assert len(dag_run.dag_versions) == 0
+ # Derive from TI versions; never return [None] from a null
created_dag_version shortcut.
+ versions = dag_run.dag_versions
+ assert isinstance(versions, list)
+ assert {dv.id for dv in versions} == ti_version_ids
+ assert None not in versions
+
+ @pytest.mark.parametrize("clear_task_ids", [["0"], ["0", "1"]],
ids=["partial", "full"])
+ def test_bundled_dag_run_dag_versions_after_run_on_latest_version(
+ self, clear_task_ids, dag_maker, session
+ ):
+ """Bundled dag_versions keep every version still on an attempt after
run_on_latest_version.
+
+ A partial clear leaves the uncleared TI on the old version. A full
clear moves every
+ current TI to latest and leaves the old version on archived attempts.
+ """
+ dag_id = (
+ "test_dag_run_dag_versions_partial_latest"
+ if clear_task_ids == ["0"]
+ else "test_dag_run_dag_versions_full_latest"
+ )
+ with dag_maker(
+ dag_id,
+ schedule=datetime.timedelta(days=1),
+ start_date=DEFAULT_DATE,
+ bundle_version="v1",
+ ):
+ EmptyOperator(task_id="0")
+ EmptyOperator(task_id="1")
+ dag_run = dag_maker.create_dagrun(state=State.RUNNING,
run_type=DagRunType.SCHEDULED)
+
+ old_dag_version = DagVersion.get_latest_version(dag_run.dag_id)
+ for ti in dag_run.task_instances:
+ ti.state = TaskInstanceState.SUCCESS
+ session.merge(ti)
+ dag_run.state = DagRunState.SUCCESS
+ session.merge(dag_run)
+ session.flush()
+
+ with dag_maker(
+ dag_id,
+ schedule=datetime.timedelta(days=1),
+ start_date=DEFAULT_DATE,
+ bundle_version="v2",
+ ):
+ EmptyOperator(task_id="0")
+ EmptyOperator(task_id="1")
+ new_dag_version = DagVersion.get_latest_version(dag_run.dag_id)
+ assert old_dag_version.id != new_dag_version.id
+
+ tis_by_id = {ti.task_id: ti for ti in dag_run.task_instances}
+ clear_task_instances(
+ [tis_by_id[task_id] for task_id in clear_task_ids],
+ session,
+ run_on_latest_version=True,
+ )
+ session.commit()
+
+ dag_run = session.get(DagRun, dag_run.id)
+ assert dag_run is not None
+ assert dag_run.created_dag_version_id == new_dag_version.id
+ assert dag_run.bundle_version == new_dag_version.bundle_version
+
+ if clear_task_ids == ["0"]:
+ tis = {ti.task_id: ti for ti in dag_run.task_instances}
+ assert tis["0"].dag_version_id == new_dag_version.id
+ assert tis["1"].dag_version_id == old_dag_version.id
+ else:
+ assert {ti.dag_version_id for ti in dag_run.task_instances} ==
{new_dag_version.id}
+ archived_version_ids = set(
+ session.scalars(
+ select(TaskInstance.dag_version_id)
+ .where(
+ TaskInstance.dag_id == dag_run.dag_id,
+ TaskInstance.run_id == dag_run.run_id,
+ TaskInstance.dag_version_id.is_not(None),
+ )
+ .execution_options(include_all_attempts=True)
+ )
+ )
+ assert old_dag_version.id in archived_version_ids
+
+ session.expire(dag_run, ["task_instances",
"historical_task_instances"])
+ dag_run._prefetched_dag_version_ids = None
+ unloaded = sa_inspect(dag_run).unloaded
+ assert "task_instances" in unloaded
+ assert "historical_task_instances" in unloaded
+ version_ids = {dv.id for dv in dag_run.dag_versions}
+ assert version_ids == {old_dag_version.id, new_dag_version.id}
+ assert "task_instances" in sa_inspect(dag_run).unloaded
+ assert "historical_task_instances" in sa_inspect(dag_run).unloaded
+
+ attach_dag_versions_to_runs([dag_run], session=session)
+ assert dag_run._prefetched_dag_version_ids is not None
+ assert set(dag_run._prefetched_dag_version_ids) ==
{old_dag_version.id, new_dag_version.id}
+ assert {dv.id for dv in dag_run.dag_versions} == {old_dag_version.id,
new_dag_version.id}
+
+ def test_dag_run_dag_versions_transient_run_uses_relationships(self):
+ """A bundled DagRun not attached to a session reads versions from
in-memory relationships.
+
+ bundle_version is set so this fails if the property falls back to
created_dag_version.
+ """
+ dag_id = "test_transient_dag_versions"
+ with DAG(dag_id=dag_id, schedule=None, start_date=DEFAULT_DATE):
+ task = EmptyOperator(task_id="empty")
+ v1 = DagVersion(id=uuid6.uuid7(), dag_id=dag_id, version_number=1,
bundle_name="testing")
+ dag_run = DagRun(dag_id=dag_id, run_id="manual",
run_type=DagRunType.MANUAL, bundle_version="v1")
+ ti = TaskInstance(task=task, run_id=dag_run.run_id,
dag_version_id=v1.id)
+ ti.dag_version = v1
+ dag_run.task_instances.append(ti)
+ assert {dv.id for dv in dag_run.dag_versions} == {v1.id}
@pytest.mark.parametrize(
"interval",