RNHTTR commented on code in PR #30375:
URL: https://github.com/apache/airflow/pull/30375#discussion_r1157632498
##########
airflow/jobs/scheduler_job.py:
##########
@@ -152,6 +152,43 @@ def __init__(
self._zombie_threshold_secs = conf.getint("scheduler",
"scheduler_zombie_task_threshold")
self._standalone_dag_processor = conf.getboolean("scheduler",
"standalone_dag_processor")
self._dag_stale_not_seen_duration = conf.getint("scheduler",
"dag_stale_not_seen_duration")
+
+ # Since the functionality for stalled_task_timeout,
task_adoption_timeout, and worker_pods_pending_timeout
+ # are now handled by a single config (task_queued_timeout), we can't
deprecate them as we normally would.
+ # So, we'll read each config and take the max value in order to ensure
we're not undercutting a legitimate
+ # use of any of these configs.
+ stalled_task_timeout = conf.getfloat("celery", "stalled_task_timeout",
fallback=0)
+ if stalled_task_timeout:
+ # TODO: Remove in Airflow 3.0
+ warnings.warn(
+ "The 'stalled_task_timeout' parameter is deprecated. "
+ "Please use 'scheduler.task_queued_timeout'.",
+ RemovedInAirflow3Warning,
+ stacklevel=2,
+ )
+ task_adoption_timeout = conf.getfloat("celery",
"task_adoption_timeout", fallback=0)
+ if task_adoption_timeout:
+ # TODO: Remove in Airflow 3.0
+ warnings.warn(
+ "The 'task_adoption_timeout' parameter is deprecated. "
+ "Please use 'scheduler.task_queued_timeout'.",
+ RemovedInAirflow3Warning,
+ stacklevel=2,
+ )
+ worker_pods_pending_timeout = conf.getfloat("kubernetes",
"worker_pods_pending_timeout", fallback=0)
+ if worker_pods_pending_timeout:
+ # TODO: Remove in Airflow 3.0
+ warnings.warn(
+ "The 'worker_pods_pending_timeout' parameter is deprecated. "
+ "Please use 'scheduler.task_queued_timeout'.",
+ RemovedInAirflow3Warning,
+ stacklevel=2,
+ )
+ task_queued_timeout = conf.getfloat("scheduler", "task_queued_timeout")
+ self._task_queued_timeout = max(
+ stalled_task_timeout, task_adoption_timeout,
worker_pods_pending_timeout, task_queued_timeout
+ )
Review Comment:
The idea to take the max is to ensure we won't step on the toes of anyone
who, for some reason, has some timeout longer than the default for
`task_queued_timeout`. cc @jedcunningham
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
