uranusjr commented on code in PR #46248:
URL: https://github.com/apache/airflow/pull/46248#discussion_r1953884418
##########
airflow/models/backfill.py:
##########
@@ -263,57 +286,54 @@ def _create_backfill_dag_run(
backfill_sort_ordinal,
session,
):
- with session.begin_nested() as nested:
- dr = session.scalar(
- with_row_locks(
- query=_get_latest_dag_run_row_query(info, session),
- session=session,
- ),
+ with session.begin_nested():
+ should_skip_create_backfill = should_create_backfill_dag_run(
+ info, reprocess_behavior, backfill_id, backfill_sort_ordinal,
session
)
- if dr:
- non_create_reason = _get_dag_run_no_create_reason(dr,
reprocess_behavior)
- if non_create_reason:
- # rolling back here restores to start of this nested tran
- # which releases the lock on the latest dag run, since we
- # are not creating a new one
- nested.rollback()
- session.add(
- BackfillDagRun(
- backfill_id=backfill_id,
- dag_run_id=None,
- logical_date=info.logical_date,
- exception_reason=non_create_reason,
- sort_ordinal=backfill_sort_ordinal,
- )
- )
- return
+ if should_skip_create_backfill:
+ return
+
dag_version = DagVersion.get_latest_version(dag.dag_id,
session=session)
- dr = dag.create_dagrun(
- run_id=dag.timetable.generate_run_id(
- run_type=DagRunType.BACKFILL_JOB,
+ try:
+ dr = dag.create_dagrun(
+ run_id=dag.timetable.generate_run_id(
+ run_type=DagRunType.BACKFILL_JOB,
+ logical_date=info.logical_date,
+ data_interval=info.data_interval,
+ ),
logical_date=info.logical_date,
data_interval=info.data_interval,
- ),
- logical_date=info.logical_date,
- data_interval=info.data_interval,
- run_after=info.run_after,
- conf=dag_run_conf,
- run_type=DagRunType.BACKFILL_JOB,
- triggered_by=DagRunTriggeredByType.BACKFILL,
- dag_version=dag_version,
- state=DagRunState.QUEUED,
- start_date=timezone.utcnow(),
- backfill_id=backfill_id,
- session=session,
- )
- session.add(
- BackfillDagRun(
+ run_after=info.run_after,
+ conf=dag_run_conf,
+ run_type=DagRunType.BACKFILL_JOB,
+ triggered_by=DagRunTriggeredByType.BACKFILL,
+ dag_version=dag_version,
+ state=DagRunState.QUEUED,
+ start_date=timezone.utcnow(),
backfill_id=backfill_id,
- dag_run_id=dr.id,
- sort_ordinal=backfill_sort_ordinal,
- logical_date=info.logical_date,
+ session=session,
+ )
+ session.add(
+ BackfillDagRun(
+ backfill_id=backfill_id,
+ dag_run_id=dr.id,
+ sort_ordinal=backfill_sort_ordinal,
+ logical_date=info.logical_date,
+ )
+ )
+ except IntegrityError:
+ log.info(
+ "Skipped creating backfill dag run for dag_id=%s
backfill_id=%s, logical_date=%s (already exists)",
+ dr.dag_id,
+ dr.id,
+ info.logical_date,
+ )
+ log.info("Doing session rollback.")
+ session.rollback()
+
+ should_create_backfill_dag_run(
Review Comment:
So the logic is (simplified)
1. Check if there’s an existing run
2. If so, return
3. Otherwise, try to create a new run
4. If that succeeds, good.
5. Otherwise, race condition? Go to `should_create_backfill_dag_run` again;
in this case, `_get_latest_dag_run_row_query` should return that conflicting
run instead, and we only insert a new BackfillDagRun without a new DagRun.
It might be better if it’s clearer that the DagRun fetched in the second
`should_create_backfill_dag_run` call should be a different one, but I’m not
exactly sure how. Maybe call `_get_latest_dag_run_row_query` in
`_create_backfill_dag_run` instead? Not sure.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
