Lee-W commented on code in PR #71072:
URL: https://github.com/apache/airflow/pull/71072#discussion_r4583748692
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -4332,6 +4357,61 @@ def _remove_unreferenced_triggers(self, *, session:
Session = NEW_SESSION) -> No
.execution_options(synchronize_session="fetch")
)
+ def _report_partition_cap_backlog(self) -> None:
+ """Log and audit the Dags in the pending partitioned Dag run backlog
above the per-tick cap."""
+ with create_session() as session:
+ backlogs_per_dag = {
+ dag_id: count
+ for dag_id, count in session.execute(
+ select(AssetPartitionDagRun.target_dag_id, func.count())
+ .select_from(AssetPartitionDagRun)
+ .join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
+ .where(*_build_pending_partitioned_apdr_filters())
+ .group_by(AssetPartitionDagRun.target_dag_id)
+ )
+ }
+ backlog_total = sum(backlogs_per_dag.values())
+ if backlog_total <= self._max_partition_dag_runs_per_loop:
+ self._partition_cap_reported_dag_ids.clear()
+ return
+
+ # No SQL-side ORDER BY: it would have no dag_id tie-break, and
relying on dict
+ # insertion order mirroring DB row order across backends (SQLite
in tests vs
+ # Postgres/MySQL in production) isn't a contract worth trusting —
sort explicitly.
Review Comment:
Removed it. It's already mentioned in the paragraph later
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]