Lee-W commented on code in PR #71072:
URL: https://github.com/apache/airflow/pull/71072#discussion_r4216942954
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -2349,8 +2363,95 @@ def _create_dagruns_for_partitioned_asset_dags(self,
session: Session) -> set[st
)
).all()
if not pending_apdrs:
+ # Backlog fully drained: the next tick that re-crosses the cap is
a new episode,
+ # not a continuation of whatever was reported before.
+ self._partition_cap_backlog_reported = False
return set()
+ sorted_dag_ids: list[str] = []
+ if len(pending_apdrs) >= self._max_partition_dag_runs_per_loop:
+ # A full fetch alone can't tell us whether that's the entire
backlog or just
+ # this tick's slice of a larger one, so we only pay for this query
then.
+ # Per-Dag counts across the *whole* backlog, not just this tick's
oldest-cap
+ # slice (`pending_apdrs`) — a Dag whose partitions haven't reached
the front of
+ # the FIFO queue yet would otherwise be missing from the log/audit
row until its
+ # turn comes up.
+ backlogs_per_dag: dict[str, int] = {
+ dag_id: count
+ for dag_id, count in session.execute(
+ select(AssetPartitionDagRun.target_dag_id, func.count())
+ .select_from(AssetPartitionDagRun)
+ .join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
+ .where(
+ AssetPartitionDagRun.created_dag_run_id.is_(None),
+ DagModel.is_stale.is_(False),
Review Comment:
The backlog count now shares its predicates with the fetch through
`_build_pending_partitioned_apdr_filters`, so paused and draining Dags are
excluded from both. The test is now parametrized over `is_stale`, `is_paused`
and `is_draining`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
