This is an automated email from the ASF dual-hosted git repository.
amoghrajesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e12aab3ac6d Discard task state by default on every task clear path
(#76806)
e12aab3ac6d is described below
commit e12aab3ac6d310572f6dd9f337aad37137c8ba1a
Author: Amogh Desai <[email protected]>
AuthorDate: Tue Nov 17 10:27:04 2026 +0530
Discard task state by default on every task clear path (#76806)
---
.../task-and-asset-state-store.rst | 4 +-
airflow-core/docs/core-concepts/dag-run.rst | 8 +-
.../docs/core-concepts/resumable-tasks.rst | 13 ++-
.../docs/core-concepts/task-state-store.rst | 5 +-
airflow-core/newsfragments/76806.significant.rst | 33 ++++++
.../api_fastapi/core_api/datamodels/dag_run.py | 5 +
.../core_api/openapi/v2-rest-api-generated.yaml | 14 +++
.../api_fastapi/core_api/routes/public/dag_run.py | 2 +
.../core_api/routes/public/task_instances.py | 10 +-
.../core_api/services/public/dag_run.py | 2 +
.../core_api/services/public/task_instances.py | 43 +-------
airflow-core/src/airflow/cli/cli_config.py | 7 ++
.../src/airflow/cli/commands/dag_command.py | 3 +
.../src/airflow/cli/commands/task_command.py | 1 +
airflow-core/src/airflow/models/taskinstance.py | 41 +++++++
.../src/airflow/serialization/definitions/dag.py | 11 ++
.../airflow/ui/openapi-gen/requests/schemas.gen.ts | 12 +++
.../airflow/ui/openapi-gen/requests/types.gen.ts | 8 ++
.../ui/src/components/Clear/Run/ClearRunButton.tsx | 2 +-
.../components/Clear/Run/ClearRunDialog.test.tsx | 102 ++++++++++++++++++
.../ui/src/components/Clear/Run/ClearRunDialog.tsx | 43 +++++---
.../pages/DagRuns/BulkClearDagRunsButton.test.tsx | 120 +++++++++++++++++++++
.../src/pages/DagRuns/BulkClearDagRunsButton.tsx | 31 +++++-
.../airflow/ui/src/queries/useBulkClearDagRuns.ts | 2 +
.../core_api/routes/public/test_dag_run.py | 39 +++++++
.../core_api/routes/public/test_task_instances.py | 9 +-
.../tests/unit/cli/commands/test_dag_command.py | 23 +++-
airflow-core/tests/unit/models/test_cleartasks.py | 53 +++++++++
.../src/airflowctl/api/datamodels/generated.py | 14 +++
airflow-ctl/src/airflowctl/ctl/cli_config.py | 7 ++
.../src/airflowctl/ctl/commands/dag_command.py | 1 +
.../airflow_ctl/ctl/commands/test_dag_command.py | 6 +-
32 files changed, 584 insertions(+), 90 deletions(-)
diff --git
a/airflow-core/docs/administration-and-deployment/task-and-asset-state-store.rst
b/airflow-core/docs/administration-and-deployment/task-and-asset-state-store.rst
index 0d88a3d28b7..31091c606c5 100644
---
a/airflow-core/docs/administration-and-deployment/task-and-asset-state-store.rst
+++
b/airflow-core/docs/administration-and-deployment/task-and-asset-state-store.rst
@@ -61,8 +61,8 @@ Number of days after which task state store rows expire. When
a key is written w
~~~~~~~~~~~~~~~~~~~~
Retention and ``clear_on_success`` are not the only ways task state store
entries get removed:
-clearing a task instance through the REST API, the UI, or ``airflowctl`` also
discards them by
-default unless ``keep_task_state`` is set. See
:doc:`/core-concepts/resumable-tasks`.
+clearing a task instance or a Dag run (REST API, UI, ``airflowctl`` or the
``airflow`` CLI) also
+discards them by default unless ``keep_task_state`` is set. See
:doc:`/core-concepts/resumable-tasks`.
When ``True``, all task state store keys for a task instance are automatically
deleted when that task instance moves to the ``success`` state. Defaults to
``False``, which preserves task state store entries after success for
observability (e.g. the submitted job ID or the last row count is still
readable from the UI or REST API after the run completes).
diff --git a/airflow-core/docs/core-concepts/dag-run.rst
b/airflow-core/docs/core-concepts/dag-run.rst
index 0ec5cc85e6b..30669dfe10d 100644
--- a/airflow-core/docs/core-concepts/dag-run.rst
+++ b/airflow-core/docs/core-concepts/dag-run.rst
@@ -258,13 +258,13 @@ There are multiple options you can select to re-run -
discarding them, so the next attempt resumes from a checkpoint or reconnects
to an external job
instead of starting over. See :doc:`resumable-tasks` for details.
-Clearing an individual task instance discards its task state store entries by
default, so the next
+Clearing a task instance or a Dag run discards task state store entries by
default, so the next
attempt starts from the beginning instead of resuming a checkpoint or
reconnecting to an external
job. See :doc:`resumable-tasks` for when to keep it instead.
-You can also clear the task through CLI using ``airflowctl tasks clear``,
which discards task state
-the same way and accepts ``--keep-task-state`` to opt back in. The legacy,
deprecated
-``airflow tasks clear`` command below always keeps task state:
+You can also clear the task through CLI using ``airflowctl tasks clear``, or
the deprecated
+``airflow tasks clear`` command below. Both discard task state the same way
and accept
+``--keep-task-state`` to opt back in:
.. code-block:: bash
diff --git a/airflow-core/docs/core-concepts/resumable-tasks.rst
b/airflow-core/docs/core-concepts/resumable-tasks.rst
index 69d267a955b..518dec00bd4 100644
--- a/airflow-core/docs/core-concepts/resumable-tasks.rst
+++ b/airflow-core/docs/core-concepts/resumable-tasks.rst
@@ -159,13 +159,12 @@ To resume from the checkpoint instead, set
``keep_task_state`` when clearing, or
corresponding box in the clear dialog. That is the right choice when nothing
about the inputs or the
code changed and you only want the task to carry on where it stopped.
-This applies to clearing individual task instances through the REST API, the
UI, or
-``airflowctl dags clear`` (which clears every task instance in the matched Dag
run(s) through this
-same endpoint). Clearing an entire Dag run through the Clear Run dialog/API,
marking a task as
-failed or success (which clears downstream tasks as a side effect), and the
core CLI's
-``airflow tasks clear`` / ``airflow dags clear`` (which call
``clear_task_instances()`` directly and
-never reach the discard endpoint) all still keep task state unconditionally
today; see
-`#72929 <https://github.com/apache/airflow/issues/72929>`_.
+This applies to every way of clearing a task: clearing task instances or whole
Dag runs through
+the REST API, the UI or ``airflowctl``, the ``airflow dags clear`` / ``airflow
tasks clear``
+commands (``--keep-task-state`` opts out), and downstream tasks cleared when a
task is marked as
+failed or success. Backfills that reprocess existing runs,
``TriggerDagRunOperator`` with
+``reset_dag_run=True``, and partition clears that also clear task instances
discard task state too,
+with no opt-out.
**Clearing a task that submitted an external job**
diff --git a/airflow-core/docs/core-concepts/task-state-store.rst
b/airflow-core/docs/core-concepts/task-state-store.rst
index 9bda390e163..ad6f6a4fa3c 100644
--- a/airflow-core/docs/core-concepts/task-state-store.rst
+++ b/airflow-core/docs/core-concepts/task-state-store.rst
@@ -322,9 +322,8 @@ To wipe state across all map indices of a task, use the
:doc:`Core API </adminis
Automatic cleanup (``clear_on_success``)
----------------------------------------
-Task state store entries are also removed when a task instance is cleared
through the REST API, the
-UI, or ``airflowctl``: clearing discards them by default so the next attempt
starts over, unless
-``keep_task_state`` is set. See :doc:`resumable-tasks` for that behaviour.
+Task state store entries are also removed when a task instance or a Dag run is
cleared: clearing
+discards them by default so the next attempt starts over, unless
``keep_task_state`` is set. See :doc:`resumable-tasks` for that behaviour.
When ``[state_store] clear_on_success = True``, all task state store keys for
a task instance are automatically deleted when the task moves to the
``success`` state. This is useful for reducing storage when post-success
observability is not needed.
diff --git a/airflow-core/newsfragments/76806.significant.rst
b/airflow-core/newsfragments/76806.significant.rst
new file mode 100644
index 00000000000..70d9f1e600d
--- /dev/null
+++ b/airflow-core/newsfragments/76806.significant.rst
@@ -0,0 +1,33 @@
+Clearing a Dag run now discards task state store entries by default
+
+Airflow 3.4.0 made clearing a task instance discard its ``task_state_store``
entries. Every other
+way of clearing a task now does the same, so the next attempt starts over
instead of resuming from
+a checkpoint or returning the result of an external job that already succeeded:
+
+* clearing a Dag run, single or bulk (REST API and UI "Clear Run" dialog)
+* ``airflow dags clear`` and ``airflow tasks clear``
+* downstream tasks cleared when a task is marked as failed or success
+* backfills that reprocess existing runs, and
``TriggerDagRunOperator(reset_dag_run=True)``
+* ``airflow partitions clear --clear-task-instances`` and the
``clearPartitions`` endpoint with
+ ``clear_task_instances``
+* ``dag.test()`` on a logical date that already has a Dag run
+
+To keep task state, pass ``keep_task_state=true`` to the Dag run clear
endpoints, tick "keep task
+state" in the Clear Run dialog, or add ``--keep-task-state`` to ``airflow dags
clear``,
+``airflow tasks clear`` or ``airflowctl dags clear``. Marking tasks, backfill,
``reset_dag_run``,
+partition clears and ``dag.test()`` have no opt-out.
+
+This also widens the permission surface of clearing a Dag run: it needs only
the Dag Run edit
+permission, but now deletes ``task_state_store`` rows that the task state
store API guards behind
+the Task Instance delete permission.
+
+* Types of change
+
+ * [ ] Dag changes
+ * [ ] Config changes
+ * [x] API changes
+ * [x] CLI changes
+ * [x] Behaviour changes
+ * [ ] Plugin changes
+ * [ ] Dependency changes
+ * [ ] Code interface changes
diff --git
a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_run.py
b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_run.py
index fec163a305a..22b41373e87 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_run.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_run.py
@@ -131,6 +131,11 @@ class BaseDAGRunClear(StrictBaseModel):
"then the ``[core] rerun_with_latest_version`` config option, "
"and finally ``False``.",
)
+ keep_task_state: bool = Field(
+ default=False,
+ description="Keep the task state store entries of the cleared task
instances so the next "
+ "attempt resumes from them. By default they are discarded, so the task
starts over.",
+ )
note: str | None = Field(default=None, max_length=1000)
@model_validator(mode="before")
diff --git
a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml
b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml
index afb27e8f61e..ecb2e2f2fdf 100644
---
a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml
+++
b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml
@@ -13869,6 +13869,13 @@ components:
after clearing. If not specified, falls back to the DAG-level
``rerun_with_latest_version``
parameter, then the ``[core] rerun_with_latest_version`` config
option,
and finally ``False``.
+ keep_task_state:
+ type: boolean
+ title: Keep Task State
+ description: Keep the task state store entries of the cleared task
instances
+ so the next attempt resumes from them. By default they are
discarded,
+ so the task starts over.
+ default: false
note:
anyOf:
- type: string
@@ -15441,6 +15448,13 @@ components:
after clearing. If not specified, falls back to the DAG-level
``rerun_with_latest_version``
parameter, then the ``[core] rerun_with_latest_version`` config
option,
and finally ``False``.
+ keep_task_state:
+ type: boolean
+ title: Keep Task State
+ description: Keep the task state store entries of the cleared task
instances
+ so the next attempt resumes from them. By default they are
discarded,
+ so the task starts over.
+ default: false
note:
anyOf:
- type: string
diff --git
a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py
b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py
index 604243ef557..1728013aac6 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py
@@ -362,6 +362,7 @@ def clear_dag_run(
only_failed=body.only_failed,
only_new=body.only_new,
run_on_latest_version=body.run_on_latest_version,
+ keep_task_state=body.keep_task_state,
note=body.note,
user=user,
)
@@ -465,6 +466,7 @@ def clear_dag_runs(
only_failed=body.only_failed,
only_new=body.only_new,
run_on_latest_version=body.run_on_latest_version,
+ keep_task_state=body.keep_task_state,
note=body.note,
user=user,
)
diff --git
a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py
b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py
index 20a58f2af9a..91ccf02d9d6 100644
---
a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py
+++
b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py
@@ -120,7 +120,6 @@ from
airflow.api_fastapi.core_api.services.public.task_instances import (
BulkTaskInstanceService,
DependentClear,
ExternalClearDepthExceeded,
- _discard_task_state_store,
_get_task_group_task_ids,
_patch_task_group_state,
_patch_task_instance_note,
@@ -1330,6 +1329,7 @@ def post_clear_task_instances(
run_on_latest_version=resolved_run_on_latest,
prevent_running_task=body.prevent_running_task,
whole_task_keys=whole_task_keys,
+ keep_task_state=body.keep_task_state,
)
task_instances = cleared
else:
@@ -1340,6 +1340,7 @@ def post_clear_task_instances(
run_on_latest_version=resolved_run_on_latest,
prevent_running_task=body.prevent_running_task,
whole_task_keys=whole_task_keys,
+ keep_task_state=body.keep_task_state,
)
if dependent_clears:
task_instances = [
@@ -1358,13 +1359,6 @@ def post_clear_task_instances(
raise
raise HTTPException(status.HTTP_409_CONFLICT, str(e)) from e
- # After the clear has succeeded, so a failed clear cannot take the
task state with it.
- # This is the only clear path that discards task state today; Dag-run
clear and
- # mark-as-failed/success (which clear downstream tasks) still keep it
unconditionally.
- # It is tracked through https://github.com/apache/airflow/issues/72929
- if not body.keep_task_state:
- _discard_task_state_store(task_instances, session,
event="Discarded task state on clear")
-
if body.note is not None:
_patch_task_instance_note(
task_instance_body=body,
diff --git
a/airflow-core/src/airflow/api_fastapi/core_api/services/public/dag_run.py
b/airflow-core/src/airflow/api_fastapi/core_api/services/public/dag_run.py
index 617ed1c86bb..de90314b1db 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/services/public/dag_run.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/services/public/dag_run.py
@@ -152,6 +152,7 @@ def perform_clear_dag_run(
run_on_latest_version: bool | None,
note: str | None,
user: BaseUser,
+ keep_task_state: bool = False,
) -> DagRun:
resolved_run_on_latest =
resolve_run_on_latest_version(run_on_latest_version, dag_id, session)
dag.clear(
@@ -160,6 +161,7 @@ def perform_clear_dag_run(
only_new=only_new,
only_failed=only_failed,
run_on_latest_version=resolved_run_on_latest,
+ keep_task_state=keep_task_state,
session=session,
)
dag_run_cleared = session.scalar(select(DagRun).where(DagRun.id ==
dag_run.id))
diff --git
a/airflow-core/src/airflow/api_fastapi/core_api/services/public/task_instances.py
b/airflow-core/src/airflow/api_fastapi/core_api/services/public/task_instances.py
index 7b1e46cab98..3c5e105e89f 100644
---
a/airflow-core/src/airflow/api_fastapi/core_api/services/public/task_instances.py
+++
b/airflow-core/src/airflow/api_fastapi/core_api/services/public/task_instances.py
@@ -90,48 +90,6 @@ MutationAction = TypeVar(
)
-def _discard_task_state_store(tis: Sequence[TI], session: Session, *, event:
str) -> None:
- """
- Discard the task state store entries of each task instance.
-
- A failure is logged and re-raised so the request fails and the session
rolls back, rather than
- reporting success while some entries survive undiscarded.
-
- This only drops the metadata DB reference row via ``_get_db_backend()``;
it does not go through
- ``get_state_backend()``. A custom ``[workers] state_store_backend``
payload is left orphaned
- with no reclaim path other than its own lifecycle/TTL policy, and a custom
``[state_store]
- backend`` is not touched at all: the worker still reads and writes there,
so a clear reports
- success while the actual state survives and a later attempt can resume
from it. Closing this
- gap needs a server-side path to the configured state backend and is
tracked for a future
- change; today, this discard is only exact for the default metastore
backend.
-
- :param event: what prompted the discard, used as the log event name.
- """
- backend = _get_db_backend()
- for ti in tis:
- scope = TaskScope(
- dag_id=ti.dag_id,
- run_id=ti.run_id,
- task_id=ti.task_id,
- region_id=ti.region_id,
- region_index=ti.region_index,
- )
- try:
- backend.clear(scope=scope, session=session)
- except Exception:
- log.warning(
- "Failed to discard task state",
- discard_event=event,
- dag_id=ti.dag_id,
- run_id=ti.run_id,
- task_id=ti.task_id,
- map_index=ti.map_index,
- exc_info=True,
- )
- raise
- log.info(event, task_instance_count=len(tis))
-
-
def _clear_task_state_store_on_success(tis: Sequence[TI], session: Session) ->
None:
"""Clear task state store rows for each TI if clear_on_success is
enabled."""
if not conf.getboolean("state_store", "clear_on_success", fallback=False):
@@ -899,6 +857,7 @@ def clear_dependent_runs(
body.run_on_latest_version, clear.dag.dag_id, session
),
prevent_running_task=body.prevent_running_task,
+ keep_task_state=body.keep_task_state,
session=session,
)
# A cleared execution continues as a new attempt with a new id, so
find the live rows again.
diff --git a/airflow-core/src/airflow/cli/cli_config.py
b/airflow-core/src/airflow/cli/cli_config.py
index 61c7c46f21e..b072a612af2 100644
--- a/airflow-core/src/airflow/cli/cli_config.py
+++ b/airflow-core/src/airflow/cli/cli_config.py
@@ -503,6 +503,11 @@ ARG_SHUT_DOWN_LOGGING = Arg(
ARG_UPSTREAM = Arg(("-u", "--upstream"), help="Include upstream tasks",
action="store_true")
ARG_ONLY_FAILED = Arg(("-f", "--only-failed"), help="Only failed jobs",
action="store_true")
ARG_ONLY_RUNNING = Arg(("-r", "--only-running"), help="Only running jobs",
action="store_true")
+ARG_KEEP_TASK_STATE = Arg(
+ ("--keep-task-state",),
+ help="Keep the task state store entries of cleared tasks so they resume
from them",
+ action="store_true",
+)
ARG_DOWNSTREAM = Arg(("-d", "--downstream"), help="Include downstream tasks",
action="store_true")
ARG_DAG_REGEX = Arg(
("-R", "--dag-regex"), help="Search dag_id as regex instead of exact
string", action="store_true"
@@ -1270,6 +1275,7 @@ DAGS_COMMANDS = (
ARG_PARTITION_DATE_END,
ARG_ONLY_FAILED,
ARG_ONLY_RUNNING,
+ ARG_KEEP_TASK_STATE,
ARG_YES,
ARG_VERBOSE,
),
@@ -1517,6 +1523,7 @@ TASKS_COMMANDS = (
ARG_YES,
ARG_ONLY_FAILED,
ARG_ONLY_RUNNING,
+ ARG_KEEP_TASK_STATE,
ARG_DAG_REGEX,
ARG_VERBOSE,
),
diff --git a/airflow-core/src/airflow/cli/commands/dag_command.py
b/airflow-core/src/airflow/cli/commands/dag_command.py
index b4dca05b227..ab1f45fdde8 100644
--- a/airflow-core/src/airflow/cli/commands/dag_command.py
+++ b/airflow-core/src/airflow/cli/commands/dag_command.py
@@ -205,6 +205,7 @@ def dag_clear(args, *, session: Session = NEW_SESSION) ->
None:
run_ids,
only_failed=args.only_failed,
only_running=args.only_running,
+ keep_task_state=args.keep_task_state,
session=session,
)
print(f"Cleared {cleared} task instance(s) across {len(run_ids)} Dag
run(s).")
@@ -216,6 +217,7 @@ def _bulk_clear_runs(
only_failed: bool,
only_running: bool,
session: Session,
+ keep_task_state: bool = False,
) -> int:
"""Clear task instances for the given run_ids in chunks instead of one
transaction per run."""
state_filter: list[TaskInstanceState] = []
@@ -239,6 +241,7 @@ def _bulk_clear_runs(
tis,
session=session,
later_loop_iterations=not state_filter,
+ keep_task_state=keep_task_state,
whole_task_keys=() if state_filter else {(ti.dag_id, ti.run_id,
ti.task_id) for ti in tis},
)
session.flush()
diff --git a/airflow-core/src/airflow/cli/commands/task_command.py
b/airflow-core/src/airflow/cli/commands/task_command.py
index e147997bd9e..e393d981411 100644
--- a/airflow-core/src/airflow/cli/commands/task_command.py
+++ b/airflow-core/src/airflow/cli/commands/task_command.py
@@ -637,6 +637,7 @@ def task_clear(args) -> None:
end_date=args.end_date,
only_failed=args.only_failed,
only_running=args.only_running,
+ keep_task_state=args.keep_task_state,
)
if not cleared:
print("Nothing to clear.")
diff --git a/airflow-core/src/airflow/models/taskinstance.py
b/airflow-core/src/airflow/models/taskinstance.py
index e1a09a82b64..60481b76da8 100644
--- a/airflow-core/src/airflow/models/taskinstance.py
+++ b/airflow-core/src/airflow/models/taskinstance.py
@@ -121,6 +121,7 @@ from airflow.models.dynamic_region import (
# Import HITLDetail at runtime so SQLAlchemy can resolve the relationship
from airflow.models.hitl import HITLDetail # noqa: F401
from airflow.models.log import Log
+from airflow.models.task_state_store import TaskStateStoreModel
from airflow.models.taskinstancekey import TaskInstanceKey
from airflow.models.taskreschedule import TaskReschedule
from airflow.models.xcom import XCOM_RETURN_KEY,
LazyXComPrimitivesSelectSequence, LazyXComSelectSequence
@@ -131,6 +132,7 @@ from airflow.ti_deps.dep_context import DepContext
from airflow.ti_deps.dependencies_deps import REQUEUEABLE_DEPS, RUNNING_DEPS
from airflow.ti_deps.deps.ready_to_reschedule import ReadyToRescheduleDep
from airflow.utils.db import exists_query
+from airflow.utils.helpers import chunks
from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.net import get_hostname
from airflow.utils.platform import getuser
@@ -150,6 +152,9 @@ TR = TaskReschedule
log = logging.getLogger(__name__)
+# Five bind parameters per key. SQLite before 3.32 allows at most 999 per
statement by default.
+_TASK_STATE_DISCARD_BATCH_SIZE = 150
+
# Columns describing one attempt's execution; a successor attempt starts with
them empty.
_PER_ATTEMPT_COLUMNS = (
"start_date",
@@ -453,6 +458,7 @@ def clear_task_instances(
*,
whole_task_keys: Collection[tuple[str, str, str]] = (),
superseded_ti_ids: Collection[UUID] = (),
+ keep_task_state: bool = False,
) -> list[TaskInstance]:
"""
Clear a set of task instances, but make sure the running ones get killed.
@@ -474,6 +480,7 @@ def clear_task_instances(
mapped expansion of such a task is archived and regenerated in a new
region.
:param superseded_ti_ids: executions whose generated work is replaced
rather than retried; they
are archived in place instead of receiving a successor.
+ :param keep_task_state: keep the ``task_state_store`` entries so the next
attempt resumes from them.
:meta private:
"""
@@ -669,9 +676,36 @@ def clear_task_instances(
select(DagRun).where(tuple_(DagRun.dag_id,
DagRun.run_id).in_(legacy_run_keys))
):
legacy_run.reconcile_legacy_expansions(session=session)
+ if not keep_task_state:
+ _discard_task_state_store(cleared, session=session)
return cleared
+def _discard_task_state_store(tis: Collection[TaskInstance], *, session:
Session) -> None:
+ """
+ Delete the ``task_state_store`` entries of each task instance.
+
+ Only the metadata DB rows are deleted. A custom ``[state_store] backend``
is not touched, and a
+ custom ``[workers] state_store_backend`` payload is left to its own
lifecycle/TTL policy.
+ """
+ columns = tuple_(
+ TaskStateStoreModel.dag_id,
+ TaskStateStoreModel.run_id,
+ TaskStateStoreModel.task_id,
+ TaskStateStoreModel.region_id,
+ TaskStateStoreModel.region_index,
+ )
+ keys = list({(ti.dag_id, ti.run_id, ti.task_id, ti.region_id,
ti.region_index) for ti in tis})
+ deleted = 0
+ for batch in chunks(keys, _TASK_STATE_DISCARD_BATCH_SIZE):
+ result = session.execute(
+
delete(TaskStateStoreModel).where(columns.in_(batch)).execution_options(synchronize_session=False)
+ )
+ deleted += getattr(result, "rowcount", 0) or 0
+ if deleted:
+ log.info("Discarded %d task state store entries for %d cleared task
instances", deleted, len(tis))
+
+
@attrs.define(frozen=True)
class LoopClearScope:
"""Live executions to retry and generated executions to archive."""
@@ -855,6 +889,7 @@ def clear_loop_task_instances(
run_on_latest_version: bool = False,
prevent_running_task: bool | None = None,
whole_task_keys: Collection[tuple[str, str, str]] = (),
+ keep_task_state: bool = False,
) -> LoopClearScope:
"""Clear selected loop tries and archive their replaced generated
suffixes."""
from airflow.models.dagrun import lock_dag_runs
@@ -881,6 +916,7 @@ def clear_loop_task_instances(
run_on_latest_version=run_on_latest_version,
prevent_running_task=prevent_running_task,
whole_task_keys=whole_task_keys,
+ keep_task_state=keep_task_state,
)
return scope
@@ -893,6 +929,7 @@ def clear_task_instances_for_runs(
run_on_latest_version: bool = False,
prevent_running_task: bool | None = None,
whole_task_keys: Collection[tuple[str, str, str]] = (),
+ keep_task_state: bool = False,
later_loop_iterations: bool = True,
) -> None:
"""
@@ -914,6 +951,7 @@ def clear_task_instances_for_runs(
run_on_latest_version=run_on_latest_version,
prevent_running_task=prevent_running_task,
whole_task_keys=whole_task_keys,
+ keep_task_state=keep_task_state,
)
for run_key in sorted(gate_runs):
clear_loop_task_instances(
@@ -925,6 +963,7 @@ def clear_task_instances_for_runs(
run_on_latest_version=run_on_latest_version,
prevent_running_task=prevent_running_task,
whole_task_keys=whole_task_keys,
+ keep_task_state=keep_task_state,
)
@@ -937,6 +976,7 @@ def apply_loop_clear_scope(
run_on_latest_version: bool = False,
prevent_running_task: bool | None = None,
whole_task_keys: Collection[tuple[str, str, str]] = (),
+ keep_task_state: bool = False,
) -> list[TaskInstance]:
"""Apply a planned clear while the caller holds its DagRun lock."""
from airflow.exceptions import AirflowClearRunningTaskException
@@ -989,6 +1029,7 @@ def apply_loop_clear_scope(
run_on_latest_version=run_on_latest_version,
prevent_running_task=prevent_running_task,
whole_task_keys=whole_task_keys,
+ keep_task_state=keep_task_state,
)
diff --git a/airflow-core/src/airflow/serialization/definitions/dag.py
b/airflow-core/src/airflow/serialization/definitions/dag.py
index c5bfb4416fb..7910a946f51 100644
--- a/airflow-core/src/airflow/serialization/definitions/dag.py
+++ b/airflow-core/src/airflow/serialization/definitions/dag.py
@@ -1153,6 +1153,7 @@ class SerializedDAG:
exclude_task_ids: frozenset[str] | frozenset[tuple[str, int]] | None =
frozenset(),
exclude_run_ids: frozenset[str] | None = frozenset(),
run_on_latest_version: bool = False,
+ keep_task_state: bool = False,
prevent_running_task: bool | None = None,
) -> set[str]: ... # pragma: no cover
@@ -1171,6 +1172,7 @@ class SerializedDAG:
exclude_task_ids: frozenset[str] | frozenset[tuple[str, int]] | None =
frozenset(),
exclude_run_ids: frozenset[str] | None = frozenset(),
run_on_latest_version: bool = False,
+ keep_task_state: bool = False,
prevent_running_task: bool | None = None,
) -> list[TaskInstance]: ... # pragma: no cover
@@ -1189,6 +1191,7 @@ class SerializedDAG:
exclude_task_ids: frozenset[str] | frozenset[tuple[str, int]] | None =
frozenset(),
exclude_run_ids: frozenset[str] | None = frozenset(),
run_on_latest_version: bool = False,
+ keep_task_state: bool = False,
prevent_running_task: bool | None = None,
) -> set[str] | list[TaskInstance]: ... # pragma: no cover
@@ -1207,6 +1210,7 @@ class SerializedDAG:
exclude_task_ids: frozenset[str] | frozenset[tuple[str, int]] | None =
frozenset(),
exclude_run_ids: frozenset[str] | None = frozenset(),
run_on_latest_version: bool = False,
+ keep_task_state: bool = False,
prevent_running_task: bool | None = None,
) -> int: ... # pragma: no cover
@@ -1225,6 +1229,7 @@ class SerializedDAG:
exclude_task_ids: frozenset[str] | frozenset[tuple[str, int]] | None =
frozenset(),
exclude_run_ids: frozenset[str] | None = frozenset(),
run_on_latest_version: bool = False,
+ keep_task_state: bool = False,
prevent_running_task: bool | None = None,
) -> list[TaskInstance]: ... # pragma: no cover
@@ -1243,6 +1248,7 @@ class SerializedDAG:
exclude_task_ids: frozenset[str] | frozenset[tuple[str, int]] | None =
frozenset(),
exclude_run_ids: frozenset[str] | None = frozenset(),
run_on_latest_version: bool = False,
+ keep_task_state: bool = False,
prevent_running_task: bool | None = None,
) -> int: ... # pragma: no cover
@@ -1263,6 +1269,7 @@ class SerializedDAG:
exclude_task_ids: frozenset[str] | frozenset[tuple[str, int]] | None =
frozenset(),
exclude_run_ids: frozenset[str] | None = frozenset(),
run_on_latest_version: bool = False,
+ keep_task_state: bool = False,
prevent_running_task: bool | None = None,
) -> int | Iterable[TaskInstance] | set[str]:
"""
@@ -1279,6 +1286,7 @@ class SerializedDAG:
be changed.
:param dry_run: Find the tasks to clear but don't clear them.
:param run_on_latest_version: whether to run on latest serialized DAG
and Bundle version
+ :param keep_task_state: keep the ``task_state_store`` entries so
cleared tasks resume from them
:param prevent_running_task: raise instead of clearing a task instance
that is running
:param session: The sqlalchemy session to use
:param exclude_task_ids: A set of ``task_id`` or (``task_id``,
``map_index``)
@@ -1378,6 +1386,7 @@ class SerializedDAG:
session=session,
dag_run_state=dag_run_state,
run_on_latest_version=run_on_latest_version,
+ keep_task_state=keep_task_state,
prevent_running_task=prevent_running_task,
later_loop_iterations=not (only_failed or only_running),
whole_task_keys={
@@ -1407,6 +1416,7 @@ class SerializedDAG:
only_running=False,
dag_run_state=DagRunState.QUEUED,
dry_run: bool = False,
+ keep_task_state: bool = False,
):
if dry_run:
tis = itertools.chain.from_iterable(
@@ -1430,6 +1440,7 @@ class SerializedDAG:
only_running=only_running,
dag_run_state=dag_run_state,
dry_run=False,
+ keep_task_state=keep_task_state,
)
for dag in dags
)
diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
index c6e92d0992e..81bbdcac12f 100644
--- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
+++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
@@ -1652,6 +1652,12 @@ export const $BulkDAGRunClearBody = {
title: 'Run On Latest Version',
description: '(Experimental) Run on the latest bundle version of
the Dag after clearing. If not specified, falls back to the DAG-level
``rerun_with_latest_version`` parameter, then the ``[core]
rerun_with_latest_version`` config option, and finally ``False``.'
},
+ keep_task_state: {
+ type: 'boolean',
+ title: 'Keep Task State',
+ description: 'Keep the task state store entries of the cleared
task instances so the next attempt resumes from them. By default they are
discarded, so the task starts over.',
+ default: false
+ },
note: {
anyOf: [
{
@@ -4027,6 +4033,12 @@ export const $DAGRunClearBody = {
title: 'Run On Latest Version',
description: '(Experimental) Run on the latest bundle version of
the Dag after clearing. If not specified, falls back to the DAG-level
``rerun_with_latest_version`` parameter, then the ``[core]
rerun_with_latest_version`` config option, and finally ``False``.'
},
+ keep_task_state: {
+ type: 'boolean',
+ title: 'Keep Task State',
+ description: 'Keep the task state store entries of the cleared
task instances so the next attempt resumes from them. By default they are
discarded, so the task starts over.',
+ default: false
+ },
note: {
anyOf: [
{
diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts
b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts
index 2b99df5aa74..39a46d2c1f9 100644
--- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts
+++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts
@@ -500,6 +500,10 @@ export type BulkDAGRunClearBody = {
* (Experimental) Run on the latest bundle version of the Dag after
clearing. If not specified, falls back to the DAG-level
``rerun_with_latest_version`` parameter, then the ``[core]
rerun_with_latest_version`` config option, and finally ``False``.
*/
run_on_latest_version?: boolean | null;
+ /**
+ * Keep the task state store entries of the cleared task instances so the
next attempt resumes from them. By default they are discarded, so the task
starts over.
+ */
+ keep_task_state?: boolean;
note?: string | null;
dag_runs?: Array<BulkDAGRunBody>;
};
@@ -1099,6 +1103,10 @@ export type DAGRunClearBody = {
* (Experimental) Run on the latest bundle version of the Dag after
clearing. If not specified, falls back to the DAG-level
``rerun_with_latest_version`` parameter, then the ``[core]
rerun_with_latest_version`` config option, and finally ``False``.
*/
run_on_latest_version?: boolean | null;
+ /**
+ * Keep the task state store entries of the cleared task instances so the
next attempt resumes from them. By default they are discarded, so the task
starts over.
+ */
+ keep_task_state?: boolean;
note?: string | null;
};
diff --git
a/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunButton.tsx
b/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunButton.tsx
index b3c8f3fafd8..c35db881942 100644
--- a/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunButton.tsx
+++ b/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunButton.tsx
@@ -60,7 +60,7 @@ export const ClearRunButton = ({ dagRun, isHotkeyEnabled =
false, ...rest }: Pro
>
<CgRedo />
</IconButton>
- <ClearRunDialog dagRun={dagRun} onClose={onClose} open={open} />
+ {open ? <ClearRunDialog dagRun={dagRun} onClose={onClose} open={open} />
: undefined}
</>
);
};
diff --git
a/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunDialog.test.tsx
b/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunDialog.test.tsx
new file mode 100644
index 00000000000..404b543646c
--- /dev/null
+++
b/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunDialog.test.tsx
@@ -0,0 +1,102 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import "@testing-library/jest-dom/vitest";
+import { fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { http, HttpResponse } from "msw";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import type { DAGRunResponse } from "openapi/requests/types.gen";
+
+import { CLEAR_KEEP_TASK_STATE_KEY } from "src/constants/localStorage";
+import { server } from "src/mocks/server";
+import { Wrapper } from "src/utils/Wrapper";
+
+import ClearRunDialog from "./ClearRunDialog";
+
+const DAG_ID = "test_dag";
+const DAG_RUN_ID = "run_1";
+
+const dagRun = {
+ bundle_version: null,
+ dag_id: DAG_ID,
+ dag_run_id: DAG_RUN_ID,
+ dag_versions: [],
+ note: null,
+} as unknown as DAGRunResponse;
+
+vi.mock("src/queries/useClearDagRunDryRun", () => ({
+ useClearDagRunDryRun: () => ({ data: { task_instances: [{ task_id: "t" }],
total_entries: 1 } }),
+ useClearDagRunDryRunKey: "clearDagRunDryRun",
+}));
+
+let clearBody: Record<string, unknown> | undefined;
+
+beforeEach(() => {
+ server.use(
+ http.get(`/api/v2/dags/${DAG_ID}/details`, () => HttpResponse.json({})),
+ http.post(`/api/v2/dags/${DAG_ID}/dagRuns/${DAG_RUN_ID}/clear`, async ({
request }) => {
+ clearBody = (await request.json()) as Record<string, unknown>;
+
+ return HttpResponse.json({ dag_id: DAG_ID, dag_run_id: DAG_RUN_ID });
+ }),
+ );
+});
+afterEach(() => {
+ clearBody = undefined;
+ localStorage.clear();
+});
+
+describe("ClearRunDialog", () => {
+ it.each([
+ { expected: false, stored: false, tick: false },
+ { expected: true, stored: false, tick: true },
+ { expected: true, stored: true, tick: false },
+ ])(
+ "sends keep_task_state=$expected with default=$stored, ticked=$tick",
+ async ({ expected, stored, tick }) => {
+ localStorage.setItem(CLEAR_KEEP_TASK_STATE_KEY, JSON.stringify(stored));
+ render(<ClearRunDialog dagRun={dagRun} onClose={vi.fn()} open />, {
wrapper: Wrapper });
+
+ const checkbox = screen.getByRole("checkbox", { name: /keepTaskState/iu
});
+
+ expect((checkbox as HTMLInputElement).checked).toBe(stored);
+ if (tick) {
+ fireEvent.click(checkbox);
+ }
+ fireEvent.click(await screen.findByRole("button", { name:
/modal\.confirm/iu }));
+
+ await waitFor(() => expect(clearBody).toBeDefined());
+ expect(clearBody?.keep_task_state).toBe(expected);
+ },
+ );
+
+ it("hides the box and sends no keep_task_state when only adding new tasks",
async () => {
+ localStorage.setItem(CLEAR_KEEP_TASK_STATE_KEY, JSON.stringify(true));
+ render(<ClearRunDialog dagRun={dagRun} onClose={vi.fn()} open />, {
wrapper: Wrapper });
+
+ fireEvent.click(await screen.findByRole("button", { name:
/options\.queueNew/iu }));
+
+ expect(screen.queryByRole("checkbox", { name: /keepTaskState/iu
})).toBeNull();
+ fireEvent.click(await screen.findByRole("button", { name:
/modal\.confirm/iu }));
+
+ await waitFor(() => expect(clearBody).toBeDefined());
+ expect(clearBody?.only_new).toBe(true);
+ expect(clearBody?.keep_task_state).toBe(false);
+ });
+});
diff --git
a/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunDialog.tsx
b/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunDialog.tsx
index dde0eb8db49..6125f9e33a0 100644
--- a/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunDialog.tsx
+++ b/airflow-core/src/airflow/ui/src/components/Clear/Run/ClearRunDialog.tsx
@@ -31,7 +31,7 @@ import { ActionAccordion } from
"src/components/ActionAccordion";
import { getRunOnLatestVersionState } from
"src/components/Clear/TaskInstance/runOnLatestVersion";
import { useRerunWithLatestVersion } from
"src/components/Clear/useRerunWithLatestVersion";
-import { useClearRunDefaultOptions } from "src/hooks/useUserSettings";
+import { useClearKeepTaskStateDefault, useClearRunDefaultOptions } from
"src/hooks/useUserSettings";
import { useClearDagRunDryRun } from "src/queries/useClearDagRunDryRun";
import { useClearDagRun } from "src/queries/useClearRun";
import { isStatePending, useAutoRefresh } from "src/utils";
@@ -48,6 +48,8 @@ const ClearRunDialog = ({ dagRun, onClose, open }: Props) => {
const { t: translate } = useTranslation();
const [note, setNote] = useState<string | null>(dagRun.note);
+ const [keepTaskStateDefault] = useClearKeepTaskStateDefault();
+ const [keepTaskState, setKeepTaskState] = useState(keepTaskStateDefault);
useEffect(() => {
if (open) {
@@ -122,6 +124,7 @@ const ClearRunDialog = ({ dagRun, onClose, open }: Props)
=> {
dagRunId,
requestBody: {
dry_run: false,
+ keep_task_state: keepTaskState && !onlyNew,
note: note === dagRun.note ? undefined : note,
only_failed: onlyFailed,
only_new: onlyNew,
@@ -134,20 +137,30 @@ const ClearRunDialog = ({ dagRun, onClose, open }: Props)
=> {
</Button>
}
footerOptions={
- shouldShowBundleVersionOption ? (
- <Checkbox
- checked={runOnLatestVersionForced || runOnLatestVersion}
- disabled={runOnLatestVersionForced}
- onCheckedChange={(event) =>
setRunOnLatestVersion(Boolean(event.checked))}
- title={
- runOnLatestVersionForced
- ?
translate("dags:runAndTaskActions.options.runOnLatestVersionForced")
- : undefined
- }
- >
- {translate("dags:runAndTaskActions.options.runOnLatestVersion")}
- </Checkbox>
- ) : undefined
+ <>
+ {onlyNew ? undefined : (
+ <Checkbox
+ checked={keepTaskState}
+ onCheckedChange={(event) =>
setKeepTaskState(Boolean(event.checked))}
+ >
+ {translate("dags:runAndTaskActions.options.keepTaskState")}
+ </Checkbox>
+ )}
+ {shouldShowBundleVersionOption ? (
+ <Checkbox
+ checked={runOnLatestVersionForced || runOnLatestVersion}
+ disabled={runOnLatestVersionForced}
+ onCheckedChange={(event) =>
setRunOnLatestVersion(Boolean(event.checked))}
+ title={
+ runOnLatestVersionForced
+ ?
translate("dags:runAndTaskActions.options.runOnLatestVersionForced")
+ : undefined
+ }
+ >
+ {translate("dags:runAndTaskActions.options.runOnLatestVersion")}
+ </Checkbox>
+ ) : undefined}
+ </>
}
lazyMount
onOpenChange={(details) => {
diff --git
a/airflow-core/src/airflow/ui/src/pages/DagRuns/BulkClearDagRunsButton.test.tsx
b/airflow-core/src/airflow/ui/src/pages/DagRuns/BulkClearDagRunsButton.test.tsx
new file mode 100644
index 00000000000..114cdfe9ac6
--- /dev/null
+++
b/airflow-core/src/airflow/ui/src/pages/DagRuns/BulkClearDagRunsButton.test.tsx
@@ -0,0 +1,120 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import "@testing-library/jest-dom/vitest";
+import { fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { http, HttpResponse } from "msw";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import type { DAGRunResponse } from "openapi/requests/types.gen";
+
+import { CLEAR_KEEP_TASK_STATE_KEY } from "src/constants/localStorage";
+import { server } from "src/mocks/server";
+import { Wrapper } from "src/utils/Wrapper";
+
+import BulkClearDagRunsButton from "./BulkClearDagRunsButton";
+
+const dagRuns = [
+ { dag_id: "test_dag", dag_run_id: "run_1" },
+ { dag_id: "test_dag", dag_run_id: "run_2" },
+] as unknown as Array<DAGRunResponse>;
+
+vi.mock("src/queries/useBulkClearDagRunsDryRun", () => ({
+ useBulkClearDagRunsDryRun: () => ({
+ data: { task_instances: [{ dag_run_id: "run_1", task_id: "t" }],
total_entries: 1 },
+ isFetching: false,
+ }),
+ useBulkClearDagRunsDryRunKey: "bulkClearDagRunsDryRun",
+}));
+
+let clearBody: Record<string, unknown> | undefined;
+
+beforeEach(() => {
+ server.use(
+ http.post("/api/v2/dags/~/clearDagRuns", async ({ request }) => {
+ clearBody = (await request.json()) as Record<string, unknown>;
+
+ return HttpResponse.json({ dag_runs: [], total_entries: 0 });
+ }),
+ );
+});
+afterEach(() => {
+ clearBody = undefined;
+ localStorage.clear();
+});
+
+describe("BulkClearDagRunsButton", () => {
+ it.each([
+ { expected: false, stored: false, tick: false },
+ { expected: true, stored: false, tick: true },
+ { expected: true, stored: true, tick: false },
+ ])(
+ "sends keep_task_state=$expected with default=$stored, ticked=$tick",
+ async ({ expected, stored, tick }) => {
+ localStorage.setItem(CLEAR_KEEP_TASK_STATE_KEY, JSON.stringify(stored));
+ render(<BulkClearDagRunsButton deselectKeys={vi.fn()}
selectedDagRuns={dagRuns} />, {
+ wrapper: Wrapper,
+ });
+
+ fireEvent.click(screen.getByRole("button", { name:
/runAndTaskActions\.clear\.button/iu }));
+ const checkbox = await screen.findByRole("checkbox", { name:
/keepTaskState/iu });
+
+ expect((checkbox as HTMLInputElement).checked).toBe(stored);
+ if (tick) {
+ fireEvent.click(checkbox);
+ }
+ fireEvent.click(await screen.findByRole("button", { name:
/modal\.confirm/iu }));
+
+ await waitFor(() => expect(clearBody).toBeDefined());
+ expect(clearBody?.keep_task_state).toBe(expected);
+ },
+ );
+
+ it("resets the box to the default when reopened after a successful clear",
async () => {
+ render(<BulkClearDagRunsButton deselectKeys={vi.fn()}
selectedDagRuns={dagRuns} />, {
+ wrapper: Wrapper,
+ });
+ const openButton = screen.getByRole("button", { name:
/runAndTaskActions\.clear\.button/iu });
+
+ fireEvent.click(openButton);
+ fireEvent.click(await screen.findByRole("checkbox", { name:
/keepTaskState/iu }));
+ fireEvent.click(await screen.findByRole("button", { name:
/modal\.confirm/iu }));
+ await waitFor(() => expect(screen.queryByRole("checkbox", { name:
/keepTaskState/iu })).toBeNull());
+
+ fireEvent.click(openButton);
+
+ expect(await screen.findByRole("checkbox", { name: /keepTaskState/iu
})).not.toBeChecked();
+ });
+
+ it("hides the box and sends no keep_task_state when only adding new tasks",
async () => {
+ localStorage.setItem(CLEAR_KEEP_TASK_STATE_KEY, JSON.stringify(true));
+ render(<BulkClearDagRunsButton deselectKeys={vi.fn()}
selectedDagRuns={dagRuns} />, {
+ wrapper: Wrapper,
+ });
+ fireEvent.click(screen.getByRole("button", { name:
/runAndTaskActions\.clear\.button/iu }));
+
+ fireEvent.click(await screen.findByRole("button", { name:
/options\.queueNew/iu }));
+
+ expect(screen.queryByRole("checkbox", { name: /keepTaskState/iu
})).toBeNull();
+ fireEvent.click(await screen.findByRole("button", { name:
/modal\.confirm/iu }));
+
+ await waitFor(() => expect(clearBody).toBeDefined());
+ expect(clearBody?.only_new).toBe(true);
+ expect(clearBody?.keep_task_state).toBe(false);
+ });
+});
diff --git
a/airflow-core/src/airflow/ui/src/pages/DagRuns/BulkClearDagRunsButton.tsx
b/airflow-core/src/airflow/ui/src/pages/DagRuns/BulkClearDagRunsButton.tsx
index 078187237bd..1a236d73969 100644
--- a/airflow-core/src/airflow/ui/src/pages/DagRuns/BulkClearDagRunsButton.tsx
+++ b/airflow-core/src/airflow/ui/src/pages/DagRuns/BulkClearDagRunsButton.tsx
@@ -24,11 +24,12 @@ import { CgRedo } from "react-icons/cg";
import type { DAGRunResponse } from "openapi/requests/types.gen";
-import { Modal, SegmentedControl } from "src/system-components";
+import { Checkbox, Modal, SegmentedControl } from "src/system-components";
import { ActionAccordion } from "src/components/ActionAccordion";
import { ActionErrors } from "src/components/ActionErrors";
+import { useClearKeepTaskStateDefault } from "src/hooks/useUserSettings";
import { useBulkClearDagRuns } from "src/queries/useBulkClearDagRuns";
import { useBulkClearDagRunsDryRun } from
"src/queries/useBulkClearDagRunsDryRun";
@@ -42,6 +43,8 @@ const BulkClearDagRunsButton = ({ deselectKeys,
selectedDagRuns }: Props) => {
const { onClose, onOpen, open } = useDisclosure();
const [selectedOptions, setSelectedOptions] =
useState<Array<string>>(["existingTasks"]);
const [note, setNote] = useState<string | null>(null);
+ const [keepTaskStateDefault] = useClearKeepTaskStateDefault();
+ const [keepTaskState, setKeepTaskState] = useState(keepTaskStateDefault);
const { bulkClear, error, isPending } = useBulkClearDagRuns({
deselectKeys,
onSuccessConfirm: onClose,
@@ -62,7 +65,14 @@ const BulkClearDagRunsButton = ({ deselectKeys,
selectedDagRuns }: Props) => {
return (
<>
- <Button onClick={onOpen} size="sm" variant="outline">
+ <Button
+ onClick={() => {
+ setKeepTaskState(keepTaskStateDefault);
+ onOpen();
+ }}
+ size="sm"
+ variant="outline"
+ >
<CgRedo />
{translate("dags:runAndTaskActions.clear.button", { type:
translate("dagRun_other") })}
</Button>
@@ -73,13 +83,28 @@ const BulkClearDagRunsButton = ({ deselectKeys,
selectedDagRuns }: Props) => {
disabled={affectedTasks.total_entries === 0}
loading={isPending || isFetching}
onClick={() => {
- bulkClear(selectedDagRuns, { note, onlyFailed, onlyNew });
+ bulkClear(selectedDagRuns, {
+ keepTaskState: keepTaskState && !onlyNew,
+ note,
+ onlyFailed,
+ onlyNew,
+ });
}}
>
<CgRedo />
{translate("modal.confirm")}
</Button>
}
+ footerOptions={
+ onlyNew ? undefined : (
+ <Checkbox
+ checked={keepTaskState}
+ onCheckedChange={(event) =>
setKeepTaskState(Boolean(event.checked))}
+ >
+ {translate("dags:runAndTaskActions.options.keepTaskState")}
+ </Checkbox>
+ )
+ }
onOpenChange={handleClose}
open={open}
size="xl"
diff --git a/airflow-core/src/airflow/ui/src/queries/useBulkClearDagRuns.ts
b/airflow-core/src/airflow/ui/src/queries/useBulkClearDagRuns.ts
index 03a29635fff..6f550b20e2a 100644
--- a/airflow-core/src/airflow/ui/src/queries/useBulkClearDagRuns.ts
+++ b/airflow-core/src/airflow/ui/src/queries/useBulkClearDagRuns.ts
@@ -42,6 +42,7 @@ type Props = {
};
export type BulkClearDagRunsOptions = {
+ keepTaskState: boolean;
note: string | null;
onlyFailed: boolean;
onlyNew: boolean;
@@ -96,6 +97,7 @@ export const useBulkClearDagRuns = ({ deselectKeys,
onSuccessConfirm }: Props) =
requestBody: {
dag_runs: dagRuns.map((dagRun) => ({ dag_id: dagRun.dag_id,
dag_run_id: dagRun.dag_run_id })),
dry_run: false,
+ keep_task_state: options.keepTaskState,
note: options.note ?? undefined,
only_failed: options.onlyFailed,
only_new: options.onlyNew,
diff --git
a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py
b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py
index 569c9102265..3cf4eaeb009 100644
--- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py
+++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py
@@ -31,6 +31,7 @@ from sqlalchemy import delete, func, insert, select, update
from airflow import plugins_manager
from airflow._shared.module_loading import qualname
+from airflow._shared.state import TaskScope
from airflow._shared.timezones import timezone
from airflow.api_fastapi.auth.managers.models.resource_details import
DagAccessEntity, DagDetails
from airflow.api_fastapi.auth.managers.simple.user import SimpleAuthManagerUser
@@ -42,12 +43,14 @@ from airflow.models import DagModel, DagRun, DagTag, Log
from airflow.models.asset import AssetEvent, AssetModel
from airflow.models.dagbundle import DagBundleModel
from airflow.models.dynamic_region import DynamicRegion
+from airflow.models.task_state_store import TaskStateStoreModel
from airflow.models.taskinstance import TaskInstance
from airflow.models.team import Team
from airflow.models.xcom import XComModel
from airflow.providers.standard.operators.empty import EmptyOperator
from airflow.sdk import Asset, Param, result, task, task_group
from airflow.sdk.definitions._internal.loop import create_loop
+from airflow.state.metastore import MetastoreBackend
from airflow.timetables.interval import CronDataIntervalTimetable
from airflow.timetables.simple import PartitionedAssetTimetable,
PartitionedAtRuntime
from airflow.timetables.trigger import CronPartitionTimetable
@@ -2316,6 +2319,41 @@ class TestClearDagRun:
logical_date=None,
)
+ @pytest.mark.usefixtures("configure_git_connection_for_dag_bundle")
+ @pytest.mark.parametrize(
+ ("url", "extra_body"),
+ [
+ pytest.param(f"/dags/{DAG1_ID}/dagRuns/{DAG1_RUN1_ID}/clear", {},
id="single"),
+ pytest.param(
+ f"/dags/{DAG1_ID}/clearDagRuns", {"dag_runs": [{"dag_run_id":
DAG1_RUN1_ID}]}, id="bulk"
+ ),
+ ],
+ )
+ @pytest.mark.parametrize("keep_task_state", [False, True])
+ def test_clear_dag_run_task_state(self, test_client, session, url,
extra_body, keep_task_state):
+ tis = session.scalars(
+ select(TaskInstance).where(TaskInstance.dag_id == DAG1_ID,
TaskInstance.run_id == DAG1_RUN1_ID)
+ ).all()
+ for ti in tis:
+ MetastoreBackend().set(
+ TaskScope(dag_id=ti.dag_id, run_id=ti.run_id,
task_id=ti.task_id, map_index=ti.map_index),
+ "job_id",
+ "app_1234",
+ session=session,
+ )
+ session.commit()
+
+ response = test_client.post(
+ url, json={"dry_run": False, "keep_task_state": keep_task_state,
**extra_body}
+ )
+ assert response.status_code == 200
+
+ session.expire_all()
+ remaining = session.scalars(
+
select(TaskStateStoreModel.task_id).where(TaskStateStoreModel.run_id ==
DAG1_RUN1_ID)
+ ).all()
+ assert sorted(remaining) == (sorted(ti.task_id for ti in tis) if
keep_task_state else [])
+
@pytest.mark.usefixtures("configure_git_connection_for_dag_bundle")
def test_clear_dag_run_whose_dag_version_was_deleted(self, test_client,
session):
"""A run that kept its bundle version after ``airflow db clean``
removed its Dag version."""
@@ -2558,6 +2596,7 @@ class TestClearDagRun:
only_new=True,
only_failed=False,
run_on_latest_version=False,
+ keep_task_state=False,
session=mock.ANY,
)
_check_last_log(
diff --git
a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_task_instances.py
b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_task_instances.py
index af71b2aab64..0936136c49b 100644
---
a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_task_instances.py
+++
b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_task_instances.py
@@ -4678,6 +4678,7 @@ class
TestPostClearTaskInstances(TestTaskInstanceEndpoint):
prevent_running_task=False,
run_on_latest_version=False,
whole_task_keys=set(),
+ keep_task_state=False,
)
def test_clear_taskinstance_is_called_with_invalid_task_ids(self,
test_client, session):
@@ -5383,9 +5384,13 @@ class
TestPostClearTaskInstances(TestTaskInstanceEndpoint):
assert self._task_state_rows(session, dag_id)
@pytest.mark.db_test
- @mock.patch.object(MetastoreBackend, "clear",
side_effect=RuntimeError("boom"))
+ @mock.patch(
+ "airflow.models.taskinstance._discard_task_state_store",
+ autospec=True,
+ side_effect=RuntimeError("boom"),
+ )
def test_clear_task_state_store_discard_failure_fails_request(self,
mock_clear, test_client, session):
- """A backend.clear() failure fails the whole request instead of
reporting a partial clear as success."""
+ """A discard failure fails the whole request instead of reporting a
partial clear as success."""
dag_id = "example_python_operator"
self.create_task_instances(
session,
diff --git a/airflow-core/tests/unit/cli/commands/test_dag_command.py
b/airflow-core/tests/unit/cli/commands/test_dag_command.py
index 8ff16d01de0..fac07bc8a3b 100644
--- a/airflow-core/tests/unit/cli/commands/test_dag_command.py
+++ b/airflow-core/tests/unit/cli/commands/test_dag_command.py
@@ -36,9 +36,10 @@ import yaml
from sqlalchemy import func, select
from airflow import settings
+from airflow._shared.state import TaskScope
from airflow._shared.timezones import timezone
from airflow.cli import cli_parser
-from airflow.cli.commands import dag_command
+from airflow.cli.commands import dag_command, task_command
from airflow.dag_processing.bundles.base import BaseDagBundle
from airflow.dag_processing.dagbag import DagBag, sync_bag_to_db
from airflow.dag_processing.processor import DagFileParsingResult,
DagFileProcessorProcess
@@ -53,6 +54,7 @@ from airflow.sdk import DAG, Asset, BaseOperator,
CronPartitionTimetable, Partit
from airflow.sdk.definitions.dag import _run_inline_trigger
from airflow.sdk.execution_time.comms import _RequestFrame, _ResponseFrame
from airflow.serialization.serialized_objects import DagSerialization,
LazyDeserializedDAG
+from airflow.state.metastore import MetastoreBackend
from airflow.timetables.base import Timetable
from airflow.triggers.base import TriggerEvent
from airflow.utils.cli import get_db_dag
@@ -1522,6 +1524,25 @@ class TestCliDagsClear:
with pytest.raises(SystemExit, match="mutually exclusive"):
dag_command.dag_clear(args)
+ @pytest.mark.usefixtures("seeded_partitioned_runs")
+ @pytest.mark.parametrize(
+ ("command", "func"),
+ [
+ (["dags", "clear", DAG_ID, "--run-id", "part_2026_03_10"],
dag_command.dag_clear),
+ (["tasks", "clear", DAG_ID], task_command.task_clear),
+ ],
+ )
+ @pytest.mark.parametrize(("extra_args", "kept"), [([], False),
(["--keep-task-state"], True)])
+ def test_keep_task_state(self, parser, command, func, extra_args, kept):
+ scope = TaskScope(dag_id=self.DAG_ID, run_id="part_2026_03_10",
task_id="t1", map_index=-1)
+ with create_session() as session:
+ MetastoreBackend().set(scope, "job_id", "app_1234",
session=session)
+
+ func(parser.parse_args([*command, "--yes", *extra_args]))
+
+ with create_session() as session:
+ assert (MetastoreBackend().get(scope, "job_id", session=session)
is not None) is kept
+
@pytest.mark.usefixtures("seeded_partitioned_runs")
def test_rejects_inverted_window(self, parser):
args = parser.parse_args(
diff --git a/airflow-core/tests/unit/models/test_cleartasks.py
b/airflow-core/tests/unit/models/test_cleartasks.py
index 5a616a340d0..8e568e20008 100644
--- a/airflow-core/tests/unit/models/test_cleartasks.py
+++ b/airflow-core/tests/unit/models/test_cleartasks.py
@@ -28,6 +28,7 @@ from sqlalchemy import delete, event, func, select, update
from sqlalchemy.orm import Session
from sqlalchemy.sql import Select
+from airflow._shared.state import TaskScope
from airflow.api_fastapi.core_api.datamodels.task_instances import
PatchTaskInstanceBody
from airflow.api_fastapi.core_api.services.public.dag_run import
perform_clear_dag_run
from airflow.api_fastapi.core_api.services.public.task_instances import
_patch_selected_task_state
@@ -39,6 +40,7 @@ from airflow.models.dagbag import DBDagBag
from airflow.models.dagrun import DagRun, clear_partition_runs
from airflow.models.dynamic_region import DynamicRegion
from airflow.models.serialized_dag import SerializedDagModel
+from airflow.models.task_state_store import TaskStateStoreModel
from airflow.models.taskinstance import (
LoopClearScope,
TaskInstance,
@@ -57,6 +59,7 @@ from airflow.sdk import task, task_group
from airflow.sdk.definitions._internal.loop import create_loop
from airflow.serialization.definitions.dag import SerializedDAG
from airflow.serialization.serialized_objects import LazyDeserializedDAG
+from airflow.state.metastore import MetastoreBackend
from airflow.ti_deps.deps.not_in_retry_period_dep import NotInRetryPeriodDep
from airflow.utils.session import create_session
from airflow.utils.state import DagRunState, State, TaskInstanceState
@@ -591,6 +594,56 @@ class TestClearTasks:
assert (successors["0"].state, successors["0"].try_number,
successors["0"].max_tries) == (None, 2, 1)
assert (successors["1"].state, successors["1"].try_number,
successors["1"].max_tries) == (None, 2, 3)
+ @pytest.mark.parametrize("keep_task_state", [False, True])
+ def test_clear_task_instances_task_state(self, dag_maker, session,
keep_task_state):
+ with dag_maker("test_clear_task_instances_task_state",
session=session):
+ EmptyOperator(task_id="cleared")
+ EmptyOperator(task_id="untouched")
+ dr = dag_maker.create_dagrun(state=State.FAILED)
+ for ti in dr.task_instances:
+ MetastoreBackend().set(
+ TaskScope(dag_id=ti.dag_id, run_id=ti.run_id,
task_id=ti.task_id, map_index=ti.map_index),
+ "job_id",
+ "app_1234",
+ session=session,
+ )
+ session.flush()
+
+ clear_task_instances(
+ [dr.get_task_instance("cleared", session=session)],
+ session=session,
+ keep_task_state=keep_task_state,
+ )
+
+ remaining = session.scalars(
+
select(TaskStateStoreModel.task_id).where(TaskStateStoreModel.run_id ==
dr.run_id)
+ ).all()
+ assert sorted(remaining) == (["cleared", "untouched"] if
keep_task_state else ["untouched"])
+
+ @mock.patch("airflow.models.taskinstance._TASK_STATE_DISCARD_BATCH_SIZE",
1)
+ def test_clear_task_instances_discards_task_state_in_batches(self,
dag_maker, session):
+ with dag_maker("test_clear_task_instances_discards_in_batches",
session=session):
+ for task_id in ("a", "b", "untouched"):
+ EmptyOperator(task_id=task_id)
+ dr = dag_maker.create_dagrun(state=State.FAILED)
+ for ti in dr.task_instances:
+ MetastoreBackend().set(
+ TaskScope(dag_id=ti.dag_id, run_id=ti.run_id,
task_id=ti.task_id, map_index=ti.map_index),
+ "job_id",
+ "app_1234",
+ session=session,
+ )
+ session.flush()
+
+ clear_task_instances(
+ [dr.get_task_instance(task_id, session=session) for task_id in
("a", "b")], session=session
+ )
+
+ remaining = session.scalars(
+
select(TaskStateStoreModel.task_id).where(TaskStateStoreModel.run_id ==
dr.run_id)
+ ).all()
+ assert remaining == ["untouched"]
+
def test_clear_task_instances_external_executor_id(self, dag_maker):
with dag_maker(
"test_clear_task_instances_external_executor_id",
diff --git a/airflow-ctl/src/airflowctl/api/datamodels/generated.py
b/airflow-ctl/src/airflowctl/api/datamodels/generated.py
index b44e53ea579..361c752e770 100644
--- a/airflow-ctl/src/airflowctl/api/datamodels/generated.py
+++ b/airflow-ctl/src/airflowctl/api/datamodels/generated.py
@@ -519,6 +519,13 @@ class DAGRunClearBody(BaseModel):
title="Run On Latest Version",
),
] = None
+ keep_task_state: Annotated[
+ bool | None,
+ Field(
+ description="Keep the task state store entries of the cleared task
instances so the next attempt resumes from them. By default they are discarded,
so the task starts over.",
+ title="Keep Task State",
+ ),
+ ] = False
note: Annotated[Note | None, Field(title="Note")] = None
@@ -2177,6 +2184,13 @@ class BulkDAGRunClearBody(BaseModel):
title="Run On Latest Version",
),
] = None
+ keep_task_state: Annotated[
+ bool | None,
+ Field(
+ description="Keep the task state store entries of the cleared task
instances so the next attempt resumes from them. By default they are discarded,
so the task starts over.",
+ title="Keep Task State",
+ ),
+ ] = False
note: Annotated[Note | None, Field(title="Note")] = None
dag_runs: Annotated[list[BulkDAGRunBody] | None, Field(title="Dag Runs")]
= None
diff --git a/airflow-ctl/src/airflowctl/ctl/cli_config.py
b/airflow-ctl/src/airflowctl/ctl/cli_config.py
index a02fac45be3..f71d6f3db03 100755
--- a/airflow-ctl/src/airflowctl/ctl/cli_config.py
+++ b/airflow-ctl/src/airflowctl/ctl/cli_config.py
@@ -390,6 +390,12 @@ ARG_DAG_CLEAR_ONLY_RUNNING = Arg(
action="store_true",
help="Only clear running task instances",
)
+ARG_DAG_CLEAR_KEEP_TASK_STATE = Arg(
+ flags=("--keep-task-state",),
+ default=False,
+ action="store_true",
+ help="Keep the task state store entries of cleared tasks so they resume
from them",
+)
ARG_DAG_CLEAR_YES = Arg(
flags=("-y", "--yes"),
default=False,
@@ -1314,6 +1320,7 @@ DAG_COMMANDS = (
ARG_DAG_PARTITION_DATE_END,
ARG_DAG_CLEAR_ONLY_FAILED,
ARG_DAG_CLEAR_ONLY_RUNNING,
+ ARG_DAG_CLEAR_KEEP_TASK_STATE,
ARG_DAG_CLEAR_YES,
),
),
diff --git a/airflow-ctl/src/airflowctl/ctl/commands/dag_command.py
b/airflow-ctl/src/airflowctl/ctl/commands/dag_command.py
index 812b3eec357..79324cf0366 100644
--- a/airflow-ctl/src/airflowctl/ctl/commands/dag_command.py
+++ b/airflow-ctl/src/airflowctl/ctl/commands/dag_command.py
@@ -294,6 +294,7 @@ def clear(args, api_client=NEW_API_CLIENT) -> dict[str, int
| bool]:
only_failed=args.only_failed,
only_running=args.only_running,
reset_dag_runs=True,
+ keep_task_state=args.keep_task_state,
),
)
cleared_task_instances += response.total_entries or 0
diff --git a/airflow-ctl/tests/airflow_ctl/ctl/commands/test_dag_command.py
b/airflow-ctl/tests/airflow_ctl/ctl/commands/test_dag_command.py
index b8b64021b4b..389aac20497 100644
--- a/airflow-ctl/tests/airflow_ctl/ctl/commands/test_dag_command.py
+++ b/airflow-ctl/tests/airflow_ctl/ctl/commands/test_dag_command.py
@@ -583,14 +583,15 @@ class TestDagCommands:
assert ctx.value is error
api_client.dag_runs.get.assert_not_called()
- def test_clear_by_run_id(self):
+ @pytest.mark.parametrize(("extra_args", "keep_task_state"), [([], False),
(["--keep-task-state"], True)])
+ def test_clear_by_run_id(self, extra_args, keep_task_state):
api_client = self._api_client_mock()
api_client.dag_runs.get.return_value =
self._dag_run("scheduled__2025-01-01")
api_client.tasks.clear.return_value = SimpleNamespace(total_entries=2)
result = dag_command.clear(
self.parser.parse_args(
- ["dags", "clear", self.dag_id, "--run-id",
"scheduled__2025-01-01", "--yes"]
+ ["dags", "clear", self.dag_id, "--run-id",
"scheduled__2025-01-01", "--yes", *extra_args]
),
api_client=api_client,
)
@@ -607,6 +608,7 @@ class TestDagCommands:
only_failed=False,
only_running=False,
reset_dag_runs=True,
+ keep_task_state=keep_task_state,
),
)