uranusjr commented on code in PR #71072:
URL: https://github.com/apache/airflow/pull/71072#discussion_r3793722661
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -2269,6 +2279,83 @@ def _create_dagruns_for_partitioned_asset_dags(self,
session: Session) -> set[st
if not pending_apdrs:
return set()
+ if len(pending_apdrs) >= self._max_partition_dag_runs_per_loop:
+ backlog_total = (
+ session.scalar(
+ select(func.count())
+ .select_from(AssetPartitionDagRun)
+ .join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
+ .where(
+ AssetPartitionDagRun.created_dag_run_id.is_(None),
+ DagModel.is_stale.is_(False),
+ )
+ )
+ or 0
+ )
+ # Distinct dag_ids across the *whole* backlog, not just this
tick's oldest-cap
+ # slice (`pending_apdrs`) — a Dag whose partitions haven't reached
the front of
+ # the FIFO queue yet would otherwise be missing from the log/audit
row until its
+ # turn comes up. Same WHERE clause as the count query above so the
two never drift.
+ backlog_dag_ids = set(
+ session.scalars(
+ select(AssetPartitionDagRun.target_dag_id)
+ .join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
+ .where(
+ AssetPartitionDagRun.created_dag_run_id.is_(None),
+ DagModel.is_stale.is_(False),
+ )
+ .distinct()
+ )
+ )
+ else:
+ backlog_total = len(pending_apdrs)
+ self._partition_cap_backlog_reported = False
+
+ if backlog_total > self._max_partition_dag_runs_per_loop:
+ self.log.warning(
+ (
+ "Reached the per-tick cap on pending partitioned Dag runs;
the remaining backlog "
+ "will be evaluated over subsequent scheduler ticks"
+ ),
+ cap=self._max_partition_dag_runs_per_loop,
+ backlog_total=backlog_total,
+ dag_ids=backlog_dag_ids,
Review Comment:
Also, more than cap pending-but-unsatisfied APDRs is a reachable steady
state; it would not change once hit until upstream keys arrive. On such a
deployment, this fires permanently on a healthy system in every scheduler tick.
I’m not sure keep warning is very friendly to users. (But we might not have a
better solution, on the other hand.)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
