uranusjr commented on code in PR #70972:
URL: https://github.com/apache/airflow/pull/70972#discussion_r3710647621
##########
airflow-core/src/airflow/assets/manager.py:
##########
@@ -827,71 +779,33 @@ def _get_or_create_apdr(
return apdr
@classmethod
- def _queue_dagruns_nonpartitioned_slow_path(
+ def _queue_dagruns_nonpartitioned(
cls, asset_id: int, dags_to_queue: set[DagModel], event: AssetEvent,
session: Session
) -> None:
- def _queue_dagrun_if_needed(dag: DagModel) -> str | None:
- item = AssetDagRunQueue(target_dag_id=dag.dag_id,
asset_id=asset_id, created_at=event.timestamp)
- # Don't error whole transaction when a single RunQueue item
conflicts.
- #
https://docs.sqlalchemy.org/en/20/orm/session_transaction.html#using-savepoint
- try:
- with session.begin_nested():
- existing = session.get(
- AssetDagRunQueue, {"target_dag_id": dag.dag_id,
"asset_id": asset_id}
- )
- if existing and existing.created_at >= event.timestamp:
- cls.logger().debug("Skipping record %s due to newer
timestamp", item)
- return dag.dag_id # already queued with a newer
timestamp
- session.merge(item)
- except exc.IntegrityError:
- cls.logger().debug("Skipping record %s", item, exc_info=True)
- return dag.dag_id
-
- queued_results = (_queue_dagrun_if_needed(dag) for dag in
dags_to_queue)
- if queued_dag_ids := [r for r in queued_results if r is not None]:
- cls.logger().debug("consuming dag ids %s", queued_dag_ids)
-
- @classmethod
- def _queue_dagruns_nonpartitioned_mysql(
- cls, asset_id: int, dags_to_queue: set[DagModel], event: AssetEvent,
session: Session
- ) -> None:
- from sqlalchemy import case
- from sqlalchemy.dialects.mysql import insert
+ if not dags_to_queue:
+ return
+ values = [
+ {"asset_id": asset_id, "target_dag_id": dag.dag_id,
"asset_event_id": event.id}
+ for dag in dags_to_queue
+ ]
- values = [{"target_dag_id": dag.dag_id} for dag in dags_to_queue]
- stmt = insert(AssetDagRunQueue).values(asset_id=asset_id,
created_at=event.timestamp)
+ if (dialect_name := get_dialect_name(session)) == "mysql":
+ from sqlalchemy.dialects.mysql import insert as my_insert
- update_stmt = stmt.on_duplicate_key_update(
- created_at=case(
- (stmt.inserted.created_at >= AssetDagRunQueue.created_at,
stmt.inserted.created_at),
- else_=AssetDagRunQueue.created_at,
- )
- )
- session.execute(update_stmt, values)
+ session.execute(my_insert(AssetDagRunQueue).prefix_with("IGNORE"),
values)
Review Comment:
Changed to on_duplicate_key_update instead. I use `asset_id=ADRQ.asset_id`
instead so the update references the original column to make it more
future-proof no-op.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
