dingo4dev commented on code in PR #62501:
URL: https://github.com/apache/airflow/pull/62501#discussion_r3522630669
##########
airflow-core/src/airflow/assets/manager.py:
##########
@@ -789,33 +819,46 @@ def _get_or_create_apdr(
return apdr
@classmethod
- def _queue_dagruns_nonpartitioned_slow_path(
- cls, asset_id: int, dags_to_queue: set[DagModel], session: Session
+ def _queue_dagruns_nonpartitioned_mysql(
+ cls, asset_id: int, dags_to_queue: set[DagModel], event: AssetEvent,
session: Session
) -> None:
- def _queue_dagrun_if_needed(dag: DagModel) -> str | None:
- item = AssetDagRunQueue(target_dag_id=dag.dag_id,
asset_id=asset_id)
- # Don't error whole transaction when a single RunQueue item
conflicts.
- #
https://docs.sqlalchemy.org/en/14/orm/session_transaction.html#using-savepoint
- try:
- with session.begin_nested():
- session.merge(item)
- except exc.IntegrityError:
- cls.logger().debug("Skipping record %s", item, exc_info=True)
- return dag.dag_id
+ from sqlalchemy import case
+ from sqlalchemy.dialects.mysql import insert
- queued_results = (_queue_dagrun_if_needed(dag) for dag in
dags_to_queue)
- if queued_dag_ids := [r for r in queued_results if r is not None]:
- cls.logger().debug("consuming dag ids %s", queued_dag_ids)
+ values = [{"target_dag_id": dag.dag_id} for dag in dags_to_queue]
+ stmt = insert(AssetDagRunQueue).values(asset_id=asset_id,
created_at=event.timestamp)
+
+ update_stmt = stmt.on_duplicate_key_update(
+ created_at=case(
+ (stmt.inserted.created_at >= AssetDagRunQueue.created_at,
stmt.inserted.created_at),
+ else_=AssetDagRunQueue.created_at,
+ )
+ )
+ session.execute(update_stmt, values)
@classmethod
- def _queue_dagruns_nonpartitioned_postgres(
- cls, asset_id: int, dags_to_queue: set[DagModel], session: Session
+ def _queue_dagruns_nonpartitioned_conflict_update(
+ cls,
+ asset_id: int,
+ dags_to_queue: set[DagModel],
+ event: AssetEvent,
+ session: Session,
+ dialect_name: str | None,
Review Comment:
I added this because the statics check (mypy-airflow-core) failed before.
maybe this code `dialect_name = get_dialect_name(session)` may return None
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
