ephraimbuddy commented on code in PR #71214:
URL: https://github.com/apache/airflow/pull/71214#discussion_r4629825202
##########
airflow-core/src/airflow/utils/db_cleanup.py:
##########
@@ -286,7 +286,8 @@ def readable_config(self):
_TableConfig(
table_name="callback",
recency_column_name="created_at",
- extra_columns=["id", "state"],
+ extra_columns=["id", "state", "dagrun_id"],
+ dag_id_scope=_IndirectDagScope(fk_column="dagrun_id",
referenced_table="dag_run"),
Review Comment:
**[P2] Preserve callbacks for excluded Dags**
Legacy triggerer callbacks retain a null `dagrun_id`, which passes the
exclusion filter. Adding callback cleanup to `-t dag_run` can therefore delete
callbacks and deadlines belonging to excluded Dags. Please backfill
deadline-backed callbacks and preserve unknown ownership during scoped cleanup.
##########
airflow-core/src/airflow/utils/db_cleanup.py:
##########
@@ -233,7 +233,7 @@ def readable_config(self):
keep_last=True,
keep_last_filters=[column("run_type") != DagRunType.MANUAL],
keep_last_group_by=["dag_id"],
- dependent_tables=["task_instance", "task_state_store", "deadline"],
+ dependent_tables=["task_instance", "task_state_store", "deadline",
"callback"],
Review Comment:
**[P2] Archive callbacks before parent deletion**
Callback cleanup skips active states and rows newer than the cutoff, but
deleting their Dag run still cascades to them without archiving. Please archive
callbacks using the selected parent IDs and cover active and newer callbacks in
the test.
##########
airflow-core/tests/unit/migrations/test_0158_callback_dagrun_id_migration.py:
##########
@@ -0,0 +1,278 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import uuid
+from io import StringIO
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import sqlalchemy as sa
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy.dialects.postgresql.psycopg import PGDialect_psycopg
+
+from airflow import settings
+
+from tests_common.test_utils.paths import AIRFLOW_CORE_SOURCES_PATH
+
+_MIGRATION_PATH = (
+ Path(AIRFLOW_CORE_SOURCES_PATH)
+ / "airflow"
+ / "migrations"
+ / "versions"
+ / "0158_3_5_0_add_dagrun_id_to_callback.py"
+)
+
+_spec = importlib.util.spec_from_file_location("migration_0158",
_MIGRATION_PATH)
+assert _spec is not None
+_migration = importlib.util.module_from_spec(_spec)
+assert _spec.loader is not None
+_spec.loader.exec_module(_migration)
+
+
+def _make_engine():
+ return sa.create_engine("sqlite:///:memory:")
+
+
+def _create_tables(conn):
+ conn.execute(sa.text("CREATE TABLE dag_run (id INTEGER PRIMARY KEY)"))
+ conn.execute(
+ sa.text(
+ """
+ CREATE TABLE callback (
+ id TEXT PRIMARY KEY,
+ type TEXT NOT NULL,
+ data TEXT NOT NULL,
+ dagrun_id INTEGER
+ )
+ """
+ )
+ )
+ conn.execute(
+ sa.text(
+ """
+ CREATE TABLE deadline (
+ id TEXT PRIMARY KEY,
+ callback_id TEXT NOT NULL,
+ dagrun_id INTEGER
+ )
+ """
+ )
+ )
+
+
+def _insert_callback(conn, callback_id: str, data: dict, callback_type: str =
"executor"):
+ conn.execute(
+ sa.text("INSERT INTO callback (id, type, data) VALUES (:id, :type,
:data)"),
+ {"id": callback_id, "type": callback_type, "data": json.dumps(data)},
+ )
+
+
+def _insert_deadline(conn, callback_id: str, dagrun_id: int):
+ conn.execute(
+ sa.text("INSERT INTO deadline (id, callback_id, dagrun_id) VALUES
(:id, :callback_id, :dagrun_id)"),
+ {"id": uuid.uuid4().hex, "callback_id": callback_id, "dagrun_id":
dagrun_id},
+ )
+
+
+def test_backfill_dagrun_id_from_legacy_callback_data():
+ engine = _make_engine()
+ extended_id = uuid.uuid4().hex
+ plain_id = uuid.uuid4().hex
+ deadline_id = uuid.uuid4().hex
+ invalid_id = uuid.uuid4().hex
+ triggerer_id = uuid.uuid4().hex
+ malformed_id = uuid.uuid4().hex
+
+ with engine.begin() as conn:
+ _create_tables(conn)
+ conn.execute(sa.text("INSERT INTO dag_run (id) VALUES (1), (2)"))
+ _insert_callback(conn, extended_id, {"__var": {"dag_run_id": "1"},
"__type": "dict"})
+ _insert_callback(conn, plain_id, {"dag_run_id": 2})
+ _insert_callback(conn, deadline_id, {"path":
"callback.without.dagrun.id"})
+ _insert_deadline(conn, deadline_id, 1)
+ _insert_callback(conn, invalid_id, {"__var": {"dag_run_id":
"missing"}, "__type": "dict"})
+ _insert_callback(conn, triggerer_id, {"__var": {"dag_run_id": "1"},
"__type": "dict"}, "triggerer")
+ conn.execute(
+ sa.text("INSERT INTO callback (id, type, data) VALUES (:id, :type,
:data)"),
+ {"id": malformed_id, "type": "executor", "data": "not-json"},
+ )
+
+ _migration._backfill_dagrun_id_from_deadline(conn, batch_size=2)
+ _migration._backfill_dagrun_id(conn, batch_size=2)
+
+ rows = conn.execute(sa.text("SELECT id, dagrun_id FROM
callback")).mappings().all()
+
+ dagrun_id_by_callback_id = {row["id"]: row["dagrun_id"] for row in rows}
+ assert dagrun_id_by_callback_id[extended_id] == 1
+ assert dagrun_id_by_callback_id[plain_id] == 2
+ assert dagrun_id_by_callback_id[deadline_id] == 1
+ assert dagrun_id_by_callback_id[invalid_id] is None
+ missing_dagrun_id = uuid.uuid4().hex
+ with engine.begin() as conn:
+ _insert_callback(conn, missing_dagrun_id, {"dag_run_id": 999})
+ _migration._backfill_dagrun_id(conn, batch_size=2)
+ assert (
+ conn.scalar(sa.text("SELECT dagrun_id FROM callback WHERE id =
:id"), {"id": missing_dagrun_id})
+ is None
+ )
+ assert dagrun_id_by_callback_id[triggerer_id] is None
+ assert dagrun_id_by_callback_id[malformed_id] is None
+
+
+def test_restore_legacy_dagrun_id_for_downgrade():
+ engine = _make_engine()
+ wrapped_id = uuid.uuid4().hex
+ plain_id = uuid.uuid4().hex
+
+ with engine.begin() as conn:
+ _create_tables(conn)
+ _insert_callback(conn, wrapped_id, {"__var": {"path":
"callback.path"}, "__type": "dict"})
+ _insert_callback(conn, plain_id, {"path": "callback.path"})
+ conn.execute(
+ sa.text("UPDATE callback SET dagrun_id = CASE id WHEN :wrapped_id
THEN 1 ELSE 2 END"),
+ {"wrapped_id": wrapped_id},
+ )
+
+ _migration._restore_legacy_dagrun_id(conn, batch_size=1)
+ rows = conn.execute(sa.text("SELECT id, data FROM
callback")).mappings().all()
+
+ data_by_callback_id = {row["id"]: json.loads(row["data"]) for row in rows}
+ assert data_by_callback_id[wrapped_id]["__var"]["dag_run_id"] == "1"
+ assert data_by_callback_id[plain_id]["dag_run_id"] == "2"
+
+
+def test_restore_legacy_dagrun_id_uses_jsonb_with_psycopg():
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("data", _migration._CALLBACK_DATA_TYPE),
+ )
+
+ statement = callback.update().where(callback.c.id ==
uuid.uuid4()).values(data={"dag_run_id": "1"})
+
+ assert "::JSONB" in str(statement.compile(dialect=PGDialect_psycopg()))
+
+
[email protected](
+ ("migration_name", "warning"),
+ [
+ (
+ "upgrade",
+ "-- WARNING: callback.dagrun_id backfill is skipped in offline
mode; "
+ "existing pending executor callbacks without dagrun_id will not
run.",
+ ),
+ (
+ "downgrade",
+ "-- WARNING: callback.data restoration of dag_run_id is skipped in
offline mode; "
+ "downgrading loses dagrun_id from existing executor callbacks.",
+ ),
+ ],
+)
+def test_offline_migration_emits_callback_data_warning(migration_name,
warning):
+ output = StringIO()
+ operations = Operations(
+ MigrationContext.configure(dialect_name="postgresql", opts={"as_sql":
True, "output_buffer": output})
+ )
+
+ with (
+ patch.object(_migration, "op", operations),
+ patch.object(_migration.context, "is_offline_mode", return_value=True),
+ ):
+ getattr(_migration, migration_name)()
+
+ assert warning in output.getvalue()
+
+
[email protected]_test
+def test_restore_legacy_dagrun_id_with_postgresql_jsonb():
+ if settings.engine.dialect.name != "postgresql":
+ pytest.skip("PostgreSQL JSONB regression test")
+
+ callback_id = uuid.uuid4()
+ with settings.engine.begin() as conn:
+ conn.execute(
+ sa.text(
+ "CREATE TEMPORARY TABLE callback "
Review Comment:
**[P2] Drop the temporary table after the test**
This temporary `callback` table survives commit on the pooled connection and
can shadow the real table in later tests. Please add `ON COMMIT DROP` to
prevent order-dependent failures.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]