kaxil commented on code in PR #61448:
URL: https://github.com/apache/airflow/pull/61448#discussion_r2892559185
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -1970,30 +1970,46 @@ def _create_dagruns_for_partitioned_asset_dags(self,
session: Session) -> set[st
partition_dag_ids.add(apdr.target_dag_id)
run_after = timezone.utcnow()
- dag_run = dag.create_dagrun(
- run_id=DagRun.generate_run_id(
- run_type=DagRunType.ASSET_TRIGGERED, logical_date=None,
run_after=run_after
- ),
- logical_date=None,
- data_interval=None,
- partition_key=apdr.partition_key,
- run_after=run_after,
- run_type=DagRunType.ASSET_TRIGGERED,
- triggered_by=DagRunTriggeredByType.ASSET,
- state=DagRunState.QUEUED,
- creating_job_id=self.job.id,
- session=session,
- )
- asset_events = session.scalars(
- select(AssetEvent).where(
- PartitionedAssetKeyLog.asset_partition_dag_run_id ==
apdr.id,
- PartitionedAssetKeyLog.asset_event_id == AssetEvent.id,
+ try:
+ dag_run = dag.create_dagrun(
+ run_id=DagRun.generate_run_id(
+ run_type=DagRunType.ASSET_TRIGGERED,
logical_date=None, run_after=run_after
+ ),
+ logical_date=None,
+ data_interval=None,
+ partition_key=apdr.partition_key,
+ run_after=run_after,
+ run_type=DagRunType.ASSET_TRIGGERED,
+ triggered_by=DagRunTriggeredByType.ASSET,
+ state=DagRunState.QUEUED,
+ creating_job_id=self.job.id,
+ session=session,
)
- )
- dag_run.consumed_asset_events.extend(asset_events)
- session.flush()
- apdr.created_dag_run_id = dag_run.id
- session.flush()
+ asset_events = session.scalars(
+ select(AssetEvent).where(
+ PartitionedAssetKeyLog.asset_partition_dag_run_id ==
apdr.id,
+ PartitionedAssetKeyLog.asset_event_id == AssetEvent.id,
+ )
+ )
+ dag_run.consumed_asset_events.extend(asset_events)
+ session.flush()
+ apdr.created_dag_run_id = dag_run.id
+ session.flush()
+ except BundleVersionUnavailable:
+ self.log.warning(
+ "Bundle version not yet available for partitioned
asset-triggered DAG %s. "
+ "Run will be created on next evaluation cycle.",
+ apdr.target_dag_id,
+ )
+ except Exception:
Review Comment:
Everything else in this PR looks good to me, apart from this major blocker:
catching broad `Exception` here can swallow `DBAPIError` and prevent the
`@retry_db_transaction` rollback/retry path from kicking in. That can leave the
session in a failed state and skip retries for transient DB failures. Could we
either (1) add an explicit `except DBAPIError: session.rollback(); raise`
before this block, or (2) rollback + re-raise DB errors, and keep only the
non-DB exceptions as log-and-continue? (Same concern applies to the similar
`except Exception` block in `_create_dag_runs_asset_triggered`.)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
