This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6cde25f1ac0 Move `[scheduler]dag_dir_list_interval` to
`[dag_bundles]refresh_interval` (#45722)
6cde25f1ac0 is described below
commit 6cde25f1ac005e217b32f19a8bf1be75bf8e7af6
Author: Jed Cunningham <[email protected]>
AuthorDate: Fri Jan 17 01:38:48 2025 -0700
Move `[scheduler]dag_dir_list_interval` to `[dag_bundles]refresh_interval`
(#45722)
This config should be the default refresh_interval in AF3, so move its
predecesor.
---
airflow/config_templates/config.yml | 16 ++++++++--------
airflow/config_templates/unit_tests.cfg | 1 -
airflow/configuration.py | 4 +++-
airflow/dag_processing/bundles/base.py | 8 +++++++-
.../administration-and-deployment/scheduler.rst | 3 ---
docs/apache-airflow/best-practices.rst | 2 +-
6 files changed, 19 insertions(+), 15 deletions(-)
diff --git a/airflow/config_templates/config.yml
b/airflow/config_templates/config.yml
index 55d4a9fbeb2..ba6af6ca11e 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -2286,13 +2286,6 @@ scheduler:
type: integer
example: ~
default: "50"
- dag_dir_list_interval:
- description: |
- How often (in seconds) to scan the DAGs directory for new files.
Default to 5 minutes.
- version_added: ~
- type: integer
- example: ~
- default: "300"
print_stats_interval:
description: |
How often should stats be printed to the logs. Setting to 0 will
disable printing stats
@@ -2681,7 +2674,7 @@ dag_bundles:
description: |
List of backend configs. Must supply name, classpath, and kwargs for
each backend.
- By default, ``refresh_interval`` is set to ``[scheduler]
dag_dir_list_interval``, but that can
+ By default, ``refresh_interval`` is set to ``[dag_bundles]
refresh_interval``, but that can
also be overridden in kwargs if desired.
The default is the dags folder dag bundle.
@@ -2711,3 +2704,10 @@ dag_bundles:
"kwargs": {{}}
}}
]
+ refresh_interval:
+ description: |
+ How often (in seconds) to refresh, or look for new files, in a DAG
bundle.
+ version_added: ~
+ type: integer
+ example: ~
+ default: "300"
diff --git a/airflow/config_templates/unit_tests.cfg
b/airflow/config_templates/unit_tests.cfg
index b29c642afe7..9e222ce5504 100644
--- a/airflow/config_templates/unit_tests.cfg
+++ b/airflow/config_templates/unit_tests.cfg
@@ -88,7 +88,6 @@ result_backend =
db+mysql://airflow:airflow@localhost:3306/airflow
job_heartbeat_sec = 1
scheduler_heartbeat_sec = 5
parsing_processes = 2
-dag_dir_list_interval = 0
[triggerer]
# Those values are set so that during unit tests things run faster than usual.
diff --git a/airflow/configuration.py b/airflow/configuration.py
index cb0cb5af73a..a0822a55278 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -324,7 +324,9 @@ class AirflowConfigParser(ConfigParser):
# A mapping of (new section, new option) -> (old section, old option,
since_version).
# When reading new option, the old option will be checked to see if it
exists. If it does a
# DeprecationWarning will be issued and the old option will be used instead
- deprecated_options: dict[tuple[str, str], tuple[str, str, str]] = {}
+ deprecated_options: dict[tuple[str, str], tuple[str, str, str]] = {
+ ("dag_bundles", "refresh_interval"): ("scheduler",
"dag_dir_list_interval", "3.0"),
+ }
# A mapping of new section -> (old section, since_version).
deprecated_sections: dict[str, tuple[str, str]] = {}
diff --git a/airflow/dag_processing/bundles/base.py
b/airflow/dag_processing/bundles/base.py
index 5ec1e9f5c34..da60f77cf4a 100644
--- a/airflow/dag_processing/bundles/base.py
+++ b/airflow/dag_processing/bundles/base.py
@@ -46,7 +46,13 @@ class BaseDagBundle(ABC):
supports_versioning: bool = False
- def __init__(self, *, name: str, refresh_interval: int = 300, version: str
| None = None) -> None:
+ def __init__(
+ self,
+ *,
+ name: str,
+ refresh_interval: int = conf.getint("dag_bundles", "refresh_interval"),
+ version: str | None = None,
+ ) -> None:
self.name = name
self.version = version
self.refresh_interval = refresh_interval
diff --git a/docs/apache-airflow/administration-and-deployment/scheduler.rst
b/docs/apache-airflow/administration-and-deployment/scheduler.rst
index 476e2e66279..d3f05738e11 100644
--- a/docs/apache-airflow/administration-and-deployment/scheduler.rst
+++ b/docs/apache-airflow/administration-and-deployment/scheduler.rst
@@ -354,9 +354,6 @@ However, you can also look at other non-performance-related
scheduler configurat
queued tasks that were launched by the dead process will be "adopted" and
monitored by this scheduler instead.
-- :ref:`config:scheduler__dag_dir_list_interval`
- How often (in seconds) to scan the DAGs directory for new files.
-
- :ref:`config:scheduler__file_parsing_sort_mode`
The scheduler will list and sort the DAG files to decide the parsing order.
diff --git a/docs/apache-airflow/best-practices.rst
b/docs/apache-airflow/best-practices.rst
index 37092f31a6d..4b25ea262ad 100644
--- a/docs/apache-airflow/best-practices.rst
+++ b/docs/apache-airflow/best-practices.rst
@@ -443,7 +443,7 @@ each parameter by following the links):
* :ref:`config:scheduler__scheduler_idle_sleep_time`
* :ref:`config:scheduler__min_file_process_interval`
-* :ref:`config:scheduler__dag_dir_list_interval`
+* :ref:`config:dag_bundles__refresh_interval`
* :ref:`config:scheduler__parsing_processes`
* :ref:`config:scheduler__file_parsing_sort_mode`
