Lee-W commented on code in PR #71072:
URL: https://github.com/apache/airflow/pull/71072#discussion_r4256563259
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -2395,8 +2421,96 @@ def _create_dagruns_for_partitioned_asset_dags(self,
session: Session) -> set[st
)
).all()
if not pending_apdrs:
+ # Backlog fully drained: the next tick that re-crosses the cap is
a new episode,
+ # not a continuation of whatever was reported before.
+ self._partition_cap_backlog_reported = False
return set()
+ sorted_dag_ids: list[str] = []
+ if (
+ len(pending_apdrs) >= self._max_partition_dag_runs_per_loop
+ or self._partition_cap_backlog_reported
+ ):
+ # A full fetch alone can't tell us whether that's the entire
backlog or just
+ # this tick's slice of a larger one, and a short fetch can't
either while other
+ # schedulers hold row locks, so we only pay for this query on a
full fetch or
+ # while an episode is being tracked.
+ # Per-Dag counts across the *whole* backlog, not just this tick's
oldest-cap
+ # slice (`pending_apdrs`) — a Dag whose partitions haven't reached
the front of
+ # the FIFO queue yet would otherwise be missing from the log/audit
row until its
+ # turn comes up.
+ backlogs_per_dag: dict[str, int] = {
+ dag_id: count
+ for dag_id, count in session.execute(
+ select(AssetPartitionDagRun.target_dag_id, func.count())
+ .select_from(AssetPartitionDagRun)
+ .join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
+ .where(*_build_pending_partitioned_apdr_filters())
+ .group_by(AssetPartitionDagRun.target_dag_id)
+ )
+ }
+ backlog_total = sum(backlogs_per_dag.values())
+ # No SQL-side ORDER BY: it would have no dag_id tie-break, and
relying on dict
+ # insertion order mirroring DB row order across backends (SQLite
in tests vs
+ # Postgres/MySQL in production) isn't a contract worth trusting —
sort explicitly.
+ sorted_dag_ids = sorted(backlogs_per_dag, key=lambda dag_id:
(-backlogs_per_dag[dag_id], dag_id))
+ if backlog_total <= self._max_partition_dag_runs_per_loop:
+ self._partition_cap_backlog_reported = False
+ else:
+ backlog_total = len(pending_apdrs)
+
+ if backlog_total > self._max_partition_dag_runs_per_loop:
+ displayed_dag_ids =
sorted_dag_ids[:MAX_PARTITION_CAP_BACKLOG_DAG_IDS_LOGGED]
+ remaining_dag_id_count = len(sorted_dag_ids) -
len(displayed_dag_ids)
+ log_cap_reached = self.log.debug if
self._partition_cap_backlog_reported else self.log.warning
Review Comment:
Updated the description: one warning per backlog eplater hits in the same
episode logged at debug.
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -2395,8 +2421,96 @@ def _create_dagruns_for_partitioned_asset_dags(self,
session: Session) -> set[st
)
).all()
if not pending_apdrs:
+ # Backlog fully drained: the next tick that re-crosses the cap is
a new episode,
+ # not a continuation of whatever was reported before.
+ self._partition_cap_backlog_reported = False
return set()
+ sorted_dag_ids: list[str] = []
+ if (
+ len(pending_apdrs) >= self._max_partition_dag_runs_per_loop
+ or self._partition_cap_backlog_reported
+ ):
+ # A full fetch alone can't tell us whether that's the entire
backlog or just
+ # this tick's slice of a larger one, and a short fetch can't
either while other
+ # schedulers hold row locks, so we only pay for this query on a
full fetch or
+ # while an episode is being tracked.
+ # Per-Dag counts across the *whole* backlog, not just this tick's
oldest-cap
+ # slice (`pending_apdrs`) — a Dag whose partitions haven't reached
the front of
+ # the FIFO queue yet would otherwise be missing from the log/audit
row until its
+ # turn comes up.
+ backlogs_per_dag: dict[str, int] = {
+ dag_id: count
+ for dag_id, count in session.execute(
+ select(AssetPartitionDagRun.target_dag_id, func.count())
+ .select_from(AssetPartitionDagRun)
+ .join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
+ .where(*_build_pending_partitioned_apdr_filters())
+ .group_by(AssetPartitionDagRun.target_dag_id)
+ )
+ }
+ backlog_total = sum(backlogs_per_dag.values())
+ # No SQL-side ORDER BY: it would have no dag_id tie-break, and
relying on dict
+ # insertion order mirroring DB row order across backends (SQLite
in tests vs
+ # Postgres/MySQL in production) isn't a contract worth trusting —
sort explicitly.
+ sorted_dag_ids = sorted(backlogs_per_dag, key=lambda dag_id:
(-backlogs_per_dag[dag_id], dag_id))
+ if backlog_total <= self._max_partition_dag_runs_per_loop:
+ self._partition_cap_backlog_reported = False
+ else:
+ backlog_total = len(pending_apdrs)
+
+ if backlog_total > self._max_partition_dag_runs_per_loop:
+ displayed_dag_ids =
sorted_dag_ids[:MAX_PARTITION_CAP_BACKLOG_DAG_IDS_LOGGED]
+ remaining_dag_id_count = len(sorted_dag_ids) -
len(displayed_dag_ids)
+ log_cap_reached = self.log.debug if
self._partition_cap_backlog_reported else self.log.warning
+ log_cap_reached(
+ (
+ "Reached the per-tick cap on pending partitioned Dag runs;
the remaining backlog "
+ "will be evaluated over subsequent scheduler ticks"
+ ),
+ cap=self._max_partition_dag_runs_per_loop,
+ backlog_total=backlog_total,
+ dag_ids=displayed_dag_ids,
+ )
+ # Edge-trigger the audit row: a persistent backlog re-hits this
branch every tick,
+ # and writing a `Log` row that often would flood the audit table.
Write it once per
+ # backlog episode; `_partition_cap_backlog_reported` is cleared
above, only when
+ # the fetch is empty or the count shows the backlog at or below
the cap.
+ if not self._partition_cap_backlog_reported:
+ # A separate, independently-committed session is required
here: the caller
+ # (`_create_dagruns_for_dags`) runs under
`@retry_db_transaction`, which rolls
+ # back *session* on a `DBAPIError` — sharing that transaction
would silently
+ # discard this audit row along with the rest of the tick's
work.
+ # Invariant: this must run before *session* makes any writes
this tick, or the
+ # new connection's commit can lock-contend with it on SQLite.
+ more_suffix = (
+ f", and {remaining_dag_id_count} more with fewer pending
runs"
+ if remaining_dag_id_count
+ else ""
+ )
+ try:
+ with create_session(scoped=False) as audit_session:
+ audit_session.add(
+ Log(
+ event="partition Dag run cap reached",
Review Comment:
The audit row now sets `owner="scheduler"` and `own"`, like
`drain_completed`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
