ramitkataria commented on code in PR #71214:
URL: https://github.com/apache/airflow/pull/71214#discussion_r4622569288
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -1453,25 +1454,19 @@ def _enqueue_executor_callbacks(self, session: Session)
-> None:
# Can't happen since we queried ExecutorCallback, but
satisfies mypy.
continue
- # TODO: Add dagrun_id as a proper ORM foreign key on the
callback table instead of storing in data dict.
- # This would eliminate this reconstruction step. For
now, all ExecutorCallbacks
- # are expected to have dag_run_id set in their data dict
(e.g., by Deadline.handle_miss).
- if not isinstance(callback.data, dict) or "dag_run_id" not in
callback.data:
+ if callback.dagrun_id is None:
self.log.error(
Review Comment:
Raising this again from [the older
thread](https://github.com/apache/airflow/pull/71214#discussion_r4331773418)
since it's outdated now: nothing sets `dagrun_id` later, so this callback stays
PENDING and is fetched and logged again on every loop. Enough of them at the
top `priority_weight` could crowd out valid callbacks. Maybe we should mark it
`FAILED` with an `output` here?
##########
airflow-core/src/airflow/models/deadline.py:
##########
@@ -327,9 +327,9 @@ def callback_data_with_context():
elif isinstance(self.callback, ExecutorCallback):
data = callback_data_with_context()
data["deadline_id"] = str(self.id)
- data["dag_run_id"] = str(self.dagrun.id)
data["dag_id"] = self.dagrun.dag_id
self.callback.data = data
+ self.callback.dagrun_id = self.dagrun.id
Review Comment:
Also from [the older
thread](https://github.com/apache/airflow/pull/71214#discussion_r4331773418):
could we set this in `Deadline.__init__`, next to `bundle_name`, instead of at
miss time? `dagrun_id` is already known there. The migration backfills every
deadline-backed executor callback regardless of state, so new SCHEDULED
callbacks would then match, and the missing-`dagrun_id` case in the scheduler
becomes unreachable for deadline callbacks.
##########
airflow-core/src/airflow/migrations/versions/0158_3_5_0_add_dagrun_id_to_callback.py:
##########
@@ -0,0 +1,217 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Add dagrun_id foreign key to callback.
+
+Revision ID: 0f4c9a2b8e1d
+Revises: c64b10719069
+Create Date: 2026-11-11 00:00:00.000000
+
+"""
+
+from __future__ import annotations
+
+import json
+
+import sqlalchemy as sa
+from alembic import context, op
+
+from airflow.configuration import conf
+from airflow.migrations.utils import disable_sqlite_fkeys
+
+# revision identifiers, used by Alembic.
+revision = "0f4c9a2b8e1d"
+down_revision = "c64b10719069"
+branch_labels = None
+depends_on = None
+airflow_version = "3.5.0"
+
+
+def _deserialize_callback_data(data):
+ if isinstance(data, str):
+ try:
+ data = json.loads(data)
+ except json.JSONDecodeError:
+ return None
+ if not isinstance(data, dict):
+ return None
+ callback_data = data.get("__var", data)
+ return (data, callback_data) if isinstance(callback_data, dict) else None
+
+
+def _extract_dagrun_id(data) -> int | None:
+ deserialized_data = _deserialize_callback_data(data)
+ if deserialized_data is None:
+ return None
+ _, callback_data = deserialized_data
+ dagrun_id = callback_data.get("dag_run_id")
+ if dagrun_id is None:
+ return None
+ try:
+ return int(dagrun_id)
+ except (TypeError, ValueError):
+ return None
+
+
+def _backfill_dagrun_id_from_deadline(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ deadline = sa.table(
+ "deadline",
+ sa.column("callback_id", sa.Uuid()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ dag_run = sa.table("dag_run", sa.column("id", sa.Integer()))
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, deadline.c.dagrun_id)
+ .select_from(
+ callback.join(deadline, callback.c.id ==
deadline.c.callback_id).join(
+ dag_run, deadline.c.dagrun_id == dag_run.c.id
+ )
+ )
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, dagrun_id in rows:
+ conn.execute(callback.update().where(callback.c.id ==
callback_id).values(dagrun_id=dagrun_id))
+
+ last_id = rows[-1].id
+
+
+def _backfill_dagrun_id(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("data", sa.Text()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ dag_run = sa.table("dag_run", sa.column("id", sa.Integer()))
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, callback.c.data)
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, data in rows:
+ dagrun_id = _extract_dagrun_id(data)
+ if dagrun_id is None:
+ continue
+
+ dagrun_exists = conn.execute(
+ sa.select(dag_run.c.id).where(dag_run.c.id ==
dagrun_id).limit(1)
+ ).first()
+ if dagrun_exists:
+ conn.execute(
+ callback.update().where(callback.c.id ==
callback_id).values(dagrun_id=dagrun_id)
+ )
+
+ last_id = rows[-1].id
+
+
+def _restore_legacy_dagrun_id(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("data", sa.Text()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, callback.c.data, callback.c.dagrun_id)
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_not(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, data, dagrun_id in rows:
+ deserialized_data = _deserialize_callback_data(data)
+ if deserialized_data is None:
+ continue
+ serialized_data, callback_data = deserialized_data
+ callback_data["dag_run_id"] = str(dagrun_id)
+ conn.execute(
+ callback.update().where(callback.c.id ==
callback_id).values(data=json.dumps(serialized_data))
Review Comment:
I think this fails on Postgres with psycopg3. `callback.data` is JSONB
there, and because the column is declared as `sa.Text()` the psycopg dialect
renders `SET data=%(data)s::VARCHAR` (psycopg2 renders no cast), which Postgres
won't assign to a jsonb column. `airflow db downgrade` would then stop at the
first executor callback with `dagrun_id` set.
Typing `data` as JSON, for example `sa.JSON().with_variant(JSONB,
"postgresql")`, and passing the dict instead of `json.dumps(...)` should avoid
it. The tests only cover SQLite, so it might be worth trying the downgrade on
Postgres with psycopg installed.
##########
airflow-core/src/airflow/migrations/versions/0158_3_5_0_add_dagrun_id_to_callback.py:
##########
@@ -0,0 +1,217 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Add dagrun_id foreign key to callback.
+
+Revision ID: 0f4c9a2b8e1d
+Revises: c64b10719069
+Create Date: 2026-11-11 00:00:00.000000
+
+"""
+
+from __future__ import annotations
+
+import json
+
+import sqlalchemy as sa
+from alembic import context, op
+
+from airflow.configuration import conf
+from airflow.migrations.utils import disable_sqlite_fkeys
+
+# revision identifiers, used by Alembic.
+revision = "0f4c9a2b8e1d"
+down_revision = "c64b10719069"
+branch_labels = None
+depends_on = None
+airflow_version = "3.5.0"
+
+
+def _deserialize_callback_data(data):
+ if isinstance(data, str):
+ try:
+ data = json.loads(data)
+ except json.JSONDecodeError:
+ return None
+ if not isinstance(data, dict):
+ return None
+ callback_data = data.get("__var", data)
+ return (data, callback_data) if isinstance(callback_data, dict) else None
+
+
+def _extract_dagrun_id(data) -> int | None:
+ deserialized_data = _deserialize_callback_data(data)
+ if deserialized_data is None:
+ return None
+ _, callback_data = deserialized_data
+ dagrun_id = callback_data.get("dag_run_id")
+ if dagrun_id is None:
+ return None
+ try:
+ return int(dagrun_id)
+ except (TypeError, ValueError):
+ return None
+
+
+def _backfill_dagrun_id_from_deadline(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ deadline = sa.table(
+ "deadline",
+ sa.column("callback_id", sa.Uuid()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ dag_run = sa.table("dag_run", sa.column("id", sa.Integer()))
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, deadline.c.dagrun_id)
+ .select_from(
+ callback.join(deadline, callback.c.id ==
deadline.c.callback_id).join(
+ dag_run, deadline.c.dagrun_id == dag_run.c.id
+ )
+ )
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, dagrun_id in rows:
+ conn.execute(callback.update().where(callback.c.id ==
callback_id).values(dagrun_id=dagrun_id))
+
+ last_id = rows[-1].id
+
+
+def _backfill_dagrun_id(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("data", sa.Text()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ dag_run = sa.table("dag_run", sa.column("id", sa.Integer()))
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, callback.c.data)
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, data in rows:
+ dagrun_id = _extract_dagrun_id(data)
+ if dagrun_id is None:
+ continue
+
+ dagrun_exists = conn.execute(
+ sa.select(dag_run.c.id).where(dag_run.c.id ==
dagrun_id).limit(1)
+ ).first()
+ if dagrun_exists:
+ conn.execute(
+ callback.update().where(callback.c.id ==
callback_id).values(dagrun_id=dagrun_id)
+ )
+
+ last_id = rows[-1].id
+
+
+def _restore_legacy_dagrun_id(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("data", sa.Text()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, callback.c.data, callback.c.dagrun_id)
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_not(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, data, dagrun_id in rows:
+ deserialized_data = _deserialize_callback_data(data)
+ if deserialized_data is None:
+ continue
+ serialized_data, callback_data = deserialized_data
+ callback_data["dag_run_id"] = str(dagrun_id)
+ conn.execute(
+ callback.update().where(callback.c.id ==
callback_id).values(data=json.dumps(serialized_data))
+ )
+
+ last_id = rows[-1].id
+
+
+def upgrade():
+ """Add callback.dagrun_id and backfill it from legacy callback.data."""
+ with disable_sqlite_fkeys(op):
+ with op.batch_alter_table("callback") as batch_op:
+ batch_op.add_column(sa.Column("dagrun_id", sa.Integer(),
nullable=True))
+
+ if not context.is_offline_mode():
Review Comment:
In offline mode, the backfill is skipped silently. Since the scheduler no
longer falls back to `data["dag_run_id"]`, executor callbacks that are pending
at upgrade time would never run. Could we print a `-- WARNING` into the
generated SQL like some other migrations? Same for the restore in downgrade
##########
airflow-core/src/airflow/utils/db_cleanup.py:
##########
@@ -233,7 +233,7 @@ def readable_config(self):
keep_last=True,
keep_last_filters=[column("run_type") != DagRunType.MANUAL],
keep_last_group_by=["dag_id"],
- dependent_tables=["task_instance", "task_state_store", "deadline"],
+ dependent_tables=["task_instance", "task_state_store", "deadline",
"callback"],
Review Comment:
Thanks for adding this. One follow-on: the `callback` config has no Dag
scope, so I think `db clean -t dag_run --dag-ids foo` now cleans old finished
callbacks for every Dag, and deleting them cascades to their `deadline` rows
without archiving those. It's the hazard the comment on the `deadline` config
describes.
Could we scope it the same way, with `dagrun_id` in `extra_columns` and
`dag_id_scope=_IndirectDagScope(fk_column="dagrun_id",
referenced_table="dag_run")`? The scope is only accurate if `dagrun_id` is set
on every deadline callback, which is another reason to set it in
`Deadline.__init__` (see my comment on `deadline.py`).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]