uranusjr commented on code in PR #70972: URL: https://github.com/apache/airflow/pull/70972#discussion_r3710656161
########## airflow-core/src/airflow/migrations/versions/0128_3_4_0_add_asset_event_id_to_adrq.py: ########## @@ -0,0 +1,196 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Reference the asset event from asset_dag_run_queue (consume-by-reference). + +Table ``asset_dag_run_queue`` gets a new column ``asset_event_id``, and the +primary key is made ``(target_dag_id, asset_event_id)``, so the scheduler +consumes queued asset events by reference instead of by a ``created_at`` time +window. ``asset_id`` is kept as a denormalized column. + +Existing rows are coalesced (one per ``(asset_id, target_dag_id)``) and carry no +event reference. Rather than dropping them (which would silently skip pending +asset-triggered Dag runs), the pre-migration scheduler's own consumption window is +replayed per dag to rebuild per-event rows: + + triggered_date = + MAX(asset_dag_run_queue.created_at) (per dag) + floor = + MAX(dag_run.run_after) for asset-triggered runs of the dag with + run_after < triggered_date (per dag) + contributing events = + the queued asset's events with + floor < asset_event.timestamp <= triggered_date + +The expansion is staged in a side table, the queue is cleared, the (now empty) +table is reshaped, and the staged rows are inserted back. + +Revision ID: b2f1a9c7d4e0 +Revises: 7a98f1b7dbd3 +Create Date: 2026-08-03 12:00:00.000000 +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from textwrap import dedent + +import sqlalchemy as sa +from alembic import context, op + +# revision identifiers, used by Alembic. +revision = "b2f1a9c7d4e0" +down_revision = "7a98f1b7dbd3" +branch_labels = None +depends_on = None +airflow_version = "3.4.0" + +_STAGING = "_adrq_migration_staging" + +_STAGE_SQL = f""" +CREATE TABLE {_STAGING} AS +SELECT DISTINCT + adrq.target_dag_id AS target_dag_id, + adrq.asset_id AS asset_id, + ae.id AS asset_event_id +FROM asset_dag_run_queue adrq +JOIN ( + SELECT t.target_dag_id, + t.triggered_date, + ( + SELECT MAX(dr.run_after) + FROM dag_run dr + WHERE dr.dag_id = t.target_dag_id + AND dr.run_type = 'asset_triggered' + AND dr.run_after < t.triggered_date + ) AS floor_date + FROM ( + SELECT target_dag_id, MAX(created_at) AS triggered_date + FROM asset_dag_run_queue + GROUP BY target_dag_id + ) t +) td ON td.target_dag_id = adrq.target_dag_id +JOIN asset_event ae + ON ae.asset_id = adrq.asset_id + AND ae.timestamp <= td.triggered_date + AND ae.timestamp > COALESCE(td.floor_date, :floor_min) Review Comment: Added a LEFT JOIN and adjusted COALESCE to cover this. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
