This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new a288a9584 BIGTOP-4360. Add Airflow to Bigtop. (#1328)
a288a9584 is described below
commit a288a958424667764eddfe8f32eca69e40e8fd1b
Author: Kengo Seki <sek...@apache.org>
AuthorDate: Fri Jul 4 13:47:50 2025 +0900
BIGTOP-4360. Add Airflow to Bigtop. (#1328)
---
bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml | 6 +-
bigtop-deploy/puppet/manifests/cluster.pp | 6 +-
.../puppet/modules/airflow/manifests/init.pp | 59 +
.../puppet/modules/airflow/templates/airflow.cfg | 2557 ++++++++++++++++++++
.../src/common/airflow/airflow-scheduler.service | 34 +
.../src/common/airflow/airflow-webserver.service | 35 +
bigtop-packages/src/common/airflow/airflow.conf | 19 +
bigtop-packages/src/common/airflow/airflow.default | 24 +
.../src/common/airflow/do-component-build | 24 +
.../src/common/airflow/install_airflow.sh | 77 +
bigtop-packages/src/deb/airflow/airflow.install | 5 +
bigtop-packages/src/deb/airflow/airflow.postinst | 34 +
bigtop-packages/src/deb/airflow/airflow.preinst | 58 +
bigtop-packages/src/deb/airflow/changelog | 1 +
bigtop-packages/src/deb/airflow/compat | 1 +
bigtop-packages/src/deb/airflow/control | 28 +
bigtop-packages/src/deb/airflow/copyright | 11 +
bigtop-packages/src/deb/airflow/rules | 37 +
bigtop-packages/src/deb/airflow/source/format | 1 +
bigtop-packages/src/rpm/airflow/SPECS/airflow.spec | 79 +
.../smoke-tests/airflow/TestAirflow.groovy | 32 +
bigtop-tests/smoke-tests/airflow/build.gradle | 34 +
bigtop.bom | 10 +
bigtop_toolchain/manifests/python.pp | 3 +
provisioner/utils/smoke-tests.sh | 1 +
25 files changed, 3174 insertions(+), 2 deletions(-)
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 3da212281..6d727b1df 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -210,6 +210,10 @@ flink::common::parallelism_default: "1"
flink::common::jobmanager_failover_strategy: "region"
flink::common::rest_port: "8081"
-
# Ranger
ranger::admin::admin_password: "Admin01234"
+
+# Airflow
+airflow::server::executor: "SequentialExecutor"
+airflow::server::load_examples: "True"
+airflow::server::sql_alchemy_conn: "sqlite:////var/lib/airflow/airflow.db"
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp
b/bigtop-deploy/puppet/manifests/cluster.pp
index 6d6af250b..177551ed6 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -121,7 +121,10 @@ $roles_map = {
},
ranger => {
master => ["ranger-server"],
- }
+ },
+ airflow => {
+ master => ["airflow"],
+ },
}
class hadoop_cluster_node (
@@ -177,6 +180,7 @@ class node_with_roles ($roles = hiera("bigtop::roles"))
inherits hadoop_cluster_
"bigtop_utils",
"phoenix",
"ranger",
+ "airflow",
]
node_with_roles::deploy_module { $modules:
diff --git a/bigtop-deploy/puppet/modules/airflow/manifests/init.pp
b/bigtop-deploy/puppet/modules/airflow/manifests/init.pp
new file mode 100644
index 000000000..1c26b3964
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/airflow/manifests/init.pp
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class airflow {
+ class deploy ($roles) {
+ if ('airflow' in $roles) {
+ include airflow::server
+ }
+ }
+
+ class server($executor, $load_examples, $sql_alchemy_conn) {
+ package { 'airflow':
+ ensure => latest,
+ }
+
+ file { '/var/lib/airflow/airflow.cfg':
+ content => template('airflow/airflow.cfg'),
+ owner => 'airflow',
+ group => 'airflow',
+ require => Package['airflow'],
+ }
+
+ exec { 'airflow-db-init':
+ command => '/usr/lib/airflow/bin/airflow db init',
+ environment => ['AIRFLOW_HOME=/var/lib/airflow'],
+ user => 'airflow',
+ require => File['/var/lib/airflow/airflow.cfg'],
+ }
+
+ exec { 'airflow-users-create':
+ command => '/usr/lib/airflow/bin/airflow users create -e
ad...@example.org -f John -l Doe -p admin -r Admin -u admin',
+ environment => ['AIRFLOW_HOME=/var/lib/airflow'],
+ user => 'airflow',
+ require => Exec['airflow-db-init'],
+ }
+
+ service { 'airflow-scheduler':
+ ensure => running,
+ require => Exec['airflow-db-init'],
+ }
+
+ service { 'airflow-webserver':
+ ensure => running,
+ require => Exec['airflow-db-init'],
+ }
+ }
+}
diff --git a/bigtop-deploy/puppet/modules/airflow/templates/airflow.cfg
b/bigtop-deploy/puppet/modules/airflow/templates/airflow.cfg
new file mode 100644
index 000000000..58bd43918
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/airflow/templates/airflow.cfg
@@ -0,0 +1,2557 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[core]
+# The folder where your airflow pipelines live, most likely a
+# subfolder in a code repository. This path must be absolute.
+#
+# Variable: AIRFLOW__CORE__DAGS_FOLDER
+#
+dags_folder = /var/lib/airflow/dags
+
+# Hostname by providing a path to a callable, which will resolve the hostname.
+# The format is "package.function".
+#
+# For example, default value ``airflow.utils.net.getfqdn`` means that result
from patched
+# version of `socket.getfqdn()
<https://docs.python.org/3/library/socket.html#socket.getfqdn>`__,
+# see related `CPython Issue
<https://github.com/python/cpython/issues/49254>`__.
+#
+# No argument should be required in the function specified.
+# If using IP address as hostname is preferred, use value
``airflow.utils.net.get_host_ip_address``
+#
+# Variable: AIRFLOW__CORE__HOSTNAME_CALLABLE
+#
+hostname_callable = airflow.utils.net.getfqdn
+
+# A callable to check if a python file has airflow dags defined or not and
should
+# return ``True`` if it has dags otherwise ``False``.
+# If this is not provided, Airflow uses its own heuristic rules.
+#
+# The function should have the following signature
+#
+# .. code-block:: python
+#
+# def func_name(file_path: str, zip_file: zipfile.ZipFile | None = None)
-> bool: ...
+#
+# Variable: AIRFLOW__CORE__MIGHT_CONTAIN_DAG_CALLABLE
+#
+might_contain_dag_callable =
airflow.utils.file.might_contain_dag_via_default_heuristic
+
+# Default timezone in case supplied date times are naive
+# can be `UTC` (default), `system`, or any `IANA
<https://www.iana.org/time-zones>`
+# timezone string (e.g. Europe/Amsterdam)
+#
+# Variable: AIRFLOW__CORE__DEFAULT_TIMEZONE
+#
+default_timezone = utc
+
+# The executor class that airflow should use. Choices include
+# ``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``,
+# ``KubernetesExecutor``, ``CeleryKubernetesExecutor``,
``LocalKubernetesExecutor`` or the
+# full import path to the class when using a custom executor.
+#
+# Variable: AIRFLOW__CORE__EXECUTOR
+#
+executor = <%= @executor %>
+
+# The auth manager class that airflow should use. Full import path to the auth
manager class.
+#
+# Variable: AIRFLOW__CORE__AUTH_MANAGER
+#
+auth_manager =
airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
+
+# This defines the maximum number of task instances that can run concurrently
per scheduler in
+# Airflow, regardless of the worker count. Generally this value, multiplied by
the number of
+# schedulers in your cluster, is the maximum number of task instances with the
running
+# state in the metadata database. Setting this value to zero allows unlimited
parallelism.
+#
+# Variable: AIRFLOW__CORE__PARALLELISM
+#
+parallelism = 32
+
+# The maximum number of task instances allowed to run concurrently in each
DAG. To calculate
+# the number of tasks that is running concurrently for a DAG, add up the
number of running
+# tasks for all DAG runs of the DAG. This is configurable at the DAG level
with ``max_active_tasks``,
+# which is defaulted as ``[core] max_active_tasks_per_dag``.
+#
+# An example scenario when this would be useful is when you want to stop a new
dag with an early
+# start date from stealing all the executor slots in a cluster.
+#
+# Variable: AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG
+#
+max_active_tasks_per_dag = 16
+
+# Are DAGs paused by default at creation
+#
+# Variable: AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION
+#
+dags_are_paused_at_creation = True
+
+# The maximum number of active DAG runs per DAG. The scheduler will not create
more DAG runs
+# if it reaches the limit. This is configurable at the DAG level with
``max_active_runs``,
+# which is defaulted as ``[core] max_active_runs_per_dag``.
+#
+# Variable: AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG
+#
+max_active_runs_per_dag = 16
+
+# (experimental) The maximum number of consecutive DAG failures before DAG is
automatically paused.
+# This is also configurable per DAG level with
``max_consecutive_failed_dag_runs``,
+# which is defaulted as ``[core] max_consecutive_failed_dag_runs_per_dag``.
+# If not specified, then the value is considered as 0,
+# meaning that the dags are never paused out by default.
+#
+# Variable: AIRFLOW__CORE__MAX_CONSECUTIVE_FAILED_DAG_RUNS_PER_DAG
+#
+max_consecutive_failed_dag_runs_per_dag = 0
+
+# The name of the method used in order to start Python processes via the
multiprocessing module.
+# This corresponds directly with the options available in the Python docs:
+# `multiprocessing.set_start_method
+#
<https://docs.python.org/3/library/multiprocessing.html#multiprocessing.set_start_method>`__
+# must be one of the values returned by
`multiprocessing.get_all_start_methods()
+#
<https://docs.python.org/3/library/multiprocessing.html#multiprocessing.get_all_start_methods>`__.
+#
+# Example: mp_start_method = fork
+#
+# Variable: AIRFLOW__CORE__MP_START_METHOD
+#
+# mp_start_method =
+
+# Whether to load the DAG examples that ship with Airflow. It's good to
+# get started, but you probably want to set this to ``False`` in a production
+# environment
+#
+# Variable: AIRFLOW__CORE__LOAD_EXAMPLES
+#
+load_examples = <%= @load_examples %>
+
+# Path to the folder containing Airflow plugins
+#
+# Variable: AIRFLOW__CORE__PLUGINS_FOLDER
+#
+plugins_folder = /var/lib/airflow/plugins
+
+# Should tasks be executed via forking of the parent process
+#
+# * ``False``: Execute via forking of the parent process
+# * ``True``: Spawning a new python process, slower than fork, but means
plugin changes picked
+# up by tasks straight away
+#
+# Variable: AIRFLOW__CORE__EXECUTE_TASKS_NEW_PYTHON_INTERPRETER
+#
+execute_tasks_new_python_interpreter = False
+
+# Secret key to save connection passwords in the db
+#
+# Variable: AIRFLOW__CORE__FERNET_KEY
+#
+fernet_key =
+
+# Whether to disable pickling dags
+#
+# Variable: AIRFLOW__CORE__DONOT_PICKLE
+#
+donot_pickle = True
+
+# How long before timing out a python file import
+#
+# Variable: AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT
+#
+dagbag_import_timeout = 30.0
+
+# Should a traceback be shown in the UI for dagbag import errors,
+# instead of just the exception message
+#
+# Variable: AIRFLOW__CORE__DAGBAG_IMPORT_ERROR_TRACEBACKS
+#
+dagbag_import_error_tracebacks = True
+
+# If tracebacks are shown, how many entries from the traceback should be shown
+#
+# Variable: AIRFLOW__CORE__DAGBAG_IMPORT_ERROR_TRACEBACK_DEPTH
+#
+dagbag_import_error_traceback_depth = 2
+
+# How long before timing out a DagFileProcessor, which processes a dag file
+#
+# Variable: AIRFLOW__CORE__DAG_FILE_PROCESSOR_TIMEOUT
+#
+dag_file_processor_timeout = 50
+
+# The class to use for running task instances in a subprocess.
+# Choices include StandardTaskRunner, CgroupTaskRunner or the full import path
to the class
+# when using a custom task runner.
+#
+# Variable: AIRFLOW__CORE__TASK_RUNNER
+#
+task_runner = StandardTaskRunner
+
+# If set, tasks without a ``run_as_user`` argument will be run with this user
+# Can be used to de-elevate a sudo user running Airflow when executing tasks
+#
+# Variable: AIRFLOW__CORE__DEFAULT_IMPERSONATION
+#
+default_impersonation =
+
+# What security module to use (for example kerberos)
+#
+# Variable: AIRFLOW__CORE__SECURITY
+#
+security =
+
+# Turn unit test mode on (overwrites many configuration options with test
+# values at runtime)
+#
+# Variable: AIRFLOW__CORE__UNIT_TEST_MODE
+#
+unit_test_mode = False
+
+# Whether to enable pickling for xcom (note that this is insecure and allows
for
+# RCE exploits).
+#
+# Variable: AIRFLOW__CORE__ENABLE_XCOM_PICKLING
+#
+enable_xcom_pickling = False
+
+# What classes can be imported during deserialization. This is a multi line
value.
+# The individual items will be parsed as a pattern to a glob function.
+# Python built-in classes (like dict) are always allowed.
+#
+# Variable: AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES
+#
+allowed_deserialization_classes = airflow.*
+
+# What classes can be imported during deserialization. This is a multi line
value.
+# The individual items will be parsed as regexp patterns.
+# This is a secondary option to ``[core] allowed_deserialization_classes``.
+#
+# Variable: AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES_REGEXP
+#
+allowed_deserialization_classes_regexp =
+
+# When a task is killed forcefully, this is the amount of time in seconds that
+# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED
+#
+# Variable: AIRFLOW__CORE__KILLED_TASK_CLEANUP_TIME
+#
+killed_task_cleanup_time = 60
+
+# Whether to override params with dag_run.conf. If you pass some key-value
pairs
+# through ``airflow dags backfill -c`` or
+# ``airflow dags trigger -c``, the key-value pairs will override the existing
ones in params.
+#
+# Variable: AIRFLOW__CORE__DAG_RUN_CONF_OVERRIDES_PARAMS
+#
+dag_run_conf_overrides_params = True
+
+# If enabled, Airflow will only scan files containing both ``DAG`` and
``airflow`` (case-insensitive).
+#
+# Variable: AIRFLOW__CORE__DAG_DISCOVERY_SAFE_MODE
+#
+dag_discovery_safe_mode = True
+
+# The pattern syntax used in the
+# `.airflowignore
+#
<https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/dags.html#airflowignore>`__
+# files in the DAG directories. Valid values are ``regexp`` or ``glob``.
+#
+# Variable: AIRFLOW__CORE__DAG_IGNORE_FILE_SYNTAX
+#
+dag_ignore_file_syntax = regexp
+
+# The number of retries each task is going to have by default. Can be
overridden at dag or task level.
+#
+# Variable: AIRFLOW__CORE__DEFAULT_TASK_RETRIES
+#
+default_task_retries = 0
+
+# The number of seconds each task is going to wait by default between retries.
Can be overridden at
+# dag or task level.
+#
+# Variable: AIRFLOW__CORE__DEFAULT_TASK_RETRY_DELAY
+#
+default_task_retry_delay = 300
+
+# The maximum delay (in seconds) each task is going to wait by default between
retries.
+# This is a global setting and cannot be overridden at task or DAG level.
+#
+# Variable: AIRFLOW__CORE__MAX_TASK_RETRY_DELAY
+#
+max_task_retry_delay = 86400
+
+# The weighting method used for the effective total priority weight of the task
+#
+# Variable: AIRFLOW__CORE__DEFAULT_TASK_WEIGHT_RULE
+#
+default_task_weight_rule = downstream
+
+# Maximum possible time (in seconds) that task will have for execution of
auxiliary processes
+# (like listeners, mini scheduler...) after task is marked as success..
+#
+# Variable: AIRFLOW__CORE__TASK_SUCCESS_OVERTIME
+#
+task_success_overtime = 20
+
+# The default task execution_timeout value for the operators. Expected an
integer value to
+# be passed into timedelta as seconds. If not specified, then the value is
considered as None,
+# meaning that the operators are never timed out by default.
+#
+# Variable: AIRFLOW__CORE__DEFAULT_TASK_EXECUTION_TIMEOUT
+#
+default_task_execution_timeout =
+
+# Updating serialized DAG can not be faster than a minimum interval to reduce
database write rate.
+#
+# Variable: AIRFLOW__CORE__MIN_SERIALIZED_DAG_UPDATE_INTERVAL
+#
+min_serialized_dag_update_interval = 30
+
+# If ``True``, serialized DAGs are compressed before writing to DB.
+#
+# .. note::
+#
+# This will disable the DAG dependencies view
+#
+# Variable: AIRFLOW__CORE__COMPRESS_SERIALIZED_DAGS
+#
+compress_serialized_dags = False
+
+# Fetching serialized DAG can not be faster than a minimum interval to reduce
database
+# read rate. This config controls when your DAGs are updated in the Webserver
+#
+# Variable: AIRFLOW__CORE__MIN_SERIALIZED_DAG_FETCH_INTERVAL
+#
+min_serialized_dag_fetch_interval = 10
+
+# Maximum number of Rendered Task Instance Fields (Template Fields) per task
to store
+# in the Database.
+# All the template_fields for each of Task Instance are stored in the Database.
+# Keeping this number small may cause an error when you try to view
``Rendered`` tab in
+# TaskInstance view for older tasks.
+#
+# Variable: AIRFLOW__CORE__MAX_NUM_RENDERED_TI_FIELDS_PER_TASK
+#
+max_num_rendered_ti_fields_per_task = 30
+
+# On each dagrun check against defined SLAs
+#
+# Variable: AIRFLOW__CORE__CHECK_SLAS
+#
+check_slas = True
+
+# Path to custom XCom class that will be used to store and resolve operators
results
+#
+# Example: xcom_backend = path.to.CustomXCom
+#
+# Variable: AIRFLOW__CORE__XCOM_BACKEND
+#
+xcom_backend = airflow.models.xcom.BaseXCom
+
+# By default Airflow plugins are lazily-loaded (only loaded when required).
Set it to ``False``,
+# if you want to load plugins whenever 'airflow' is invoked via cli or loaded
from module.
+#
+# Variable: AIRFLOW__CORE__LAZY_LOAD_PLUGINS
+#
+lazy_load_plugins = True
+
+# By default Airflow providers are lazily-discovered (discovery and imports
happen only when required).
+# Set it to ``False``, if you want to discover providers whenever 'airflow' is
invoked via cli or
+# loaded from module.
+#
+# Variable: AIRFLOW__CORE__LAZY_DISCOVER_PROVIDERS
+#
+lazy_discover_providers = True
+
+# Hide sensitive **Variables** or **Connection extra json keys** from UI
+# and task logs when set to ``True``
+#
+# .. note::
+#
+# Connection passwords are always hidden in logs
+#
+# Variable: AIRFLOW__CORE__HIDE_SENSITIVE_VAR_CONN_FIELDS
+#
+hide_sensitive_var_conn_fields = True
+
+# A comma-separated list of extra sensitive keywords to look for in variables
names or connection's
+# extra JSON.
+#
+# Variable: AIRFLOW__CORE__SENSITIVE_VAR_CONN_NAMES
+#
+sensitive_var_conn_names =
+
+# Task Slot counts for ``default_pool``. This setting would not have any
effect in an existing
+# deployment where the ``default_pool`` is already created. For existing
deployments, users can
+# change the number of slots using Webserver, API or the CLI
+#
+# Variable: AIRFLOW__CORE__DEFAULT_POOL_TASK_SLOT_COUNT
+#
+default_pool_task_slot_count = 128
+
+# The maximum list/dict length an XCom can push to trigger task mapping. If
the pushed list/dict has a
+# length exceeding this value, the task pushing the XCom will be failed
automatically to prevent the
+# mapped tasks from clogging the scheduler.
+#
+# Variable: AIRFLOW__CORE__MAX_MAP_LENGTH
+#
+max_map_length = 1024
+
+# The default umask to use for process when run in daemon mode (scheduler,
worker, etc.)
+#
+# This controls the file-creation mode mask which determines the initial value
of file permission bits
+# for newly created files.
+#
+# This value is treated as an octal-integer.
+#
+# Variable: AIRFLOW__CORE__DAEMON_UMASK
+#
+daemon_umask = 0o077
+
+# Class to use as dataset manager.
+#
+# Example: dataset_manager_class = airflow.datasets.manager.DatasetManager
+#
+# Variable: AIRFLOW__CORE__DATASET_MANAGER_CLASS
+#
+# dataset_manager_class =
+
+# Kwargs to supply to dataset manager.
+#
+# Example: dataset_manager_kwargs = {"some_param": "some_value"}
+#
+# Variable: AIRFLOW__CORE__DATASET_MANAGER_KWARGS
+#
+# dataset_manager_kwargs =
+
+# Dataset URI validation should raise an exception if it is not compliant with
AIP-60.
+# By default this configuration is false, meaning that Airflow 2.x only warns
the user.
+# In Airflow 3, this configuration will be removed, unconditionally enabling
strict validation.
+#
+# Variable: AIRFLOW__CORE__STRICT_DATASET_URI_VALIDATION
+#
+strict_dataset_uri_validation = False
+
+# (experimental) Whether components should use Airflow Internal API for DB
connectivity.
+#
+# Variable: AIRFLOW__CORE__DATABASE_ACCESS_ISOLATION
+#
+database_access_isolation = False
+
+# (experimental) Airflow Internal API url.
+# Only used if ``[core] database_access_isolation`` is ``True``.
+#
+# Example: internal_api_url = http://localhost:8080
+#
+# Variable: AIRFLOW__CORE__INTERNAL_API_URL
+#
+# internal_api_url =
+
+# Secret key used to authenticate internal API clients to core. It should be
as random as possible.
+# However, when running more than 1 instances of webserver / internal API
services, make sure all
+# of them use the same ``secret_key`` otherwise calls will fail on
authentication.
+# The authentication token generated using the secret key has a short expiry
time though - make
+# sure that time on ALL the machines that you run airflow components on is
synchronized
+# (for example using ntpd) otherwise you might get "forbidden" errors when the
logs are accessed.
+#
+# Variable: AIRFLOW__CORE__INTERNAL_API_SECRET_KEY
+#
+internal_api_secret_key = dAtBycpTM5FZ3iBrvRjUGA==
+
+# The ability to allow testing connections across Airflow UI, API and CLI.
+# Supported options: ``Disabled``, ``Enabled``, ``Hidden``. Default: Disabled
+# Disabled - Disables the test connection functionality and disables the Test
Connection button in UI.
+# Enabled - Enables the test connection functionality and shows the Test
Connection button in UI.
+# Hidden - Disables the test connection functionality and hides the Test
Connection button in UI.
+# Before setting this to Enabled, make sure that you review the users who are
able to add/edit
+# connections and ensure they are trusted. Connection testing can be done
maliciously leading to
+# undesired and insecure outcomes.
+# See `Airflow Security Model: Capabilities of authenticated UI users
+#
<https://airflow.apache.org/docs/apache-airflow/stable/security/security_model.html#capabilities-of-authenticated-ui-users>`__
+# for more details.
+#
+# Variable: AIRFLOW__CORE__TEST_CONNECTION
+#
+test_connection = Disabled
+
+# The maximum length of the rendered template field. If the value to be stored
in the
+# rendered template field exceeds this size, it's redacted.
+#
+# Variable: AIRFLOW__CORE__MAX_TEMPLATED_FIELD_LENGTH
+#
+max_templated_field_length = 4096
+
+[database]
+# Path to the ``alembic.ini`` file. You can either provide the file path
relative
+# to the Airflow home directory or the absolute path if it is located
elsewhere.
+#
+# Variable: AIRFLOW__DATABASE__ALEMBIC_INI_FILE_PATH
+#
+alembic_ini_file_path = alembic.ini
+
+# The SQLAlchemy connection string to the metadata database.
+# SQLAlchemy supports many different database engines.
+# See: `Set up a Database Backend: Database URI
+#
<https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html#database-uri>`__
+# for more details.
+#
+# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
+#
+sql_alchemy_conn = <%= @sql_alchemy_conn %>
+
+# Extra engine specific keyword args passed to SQLAlchemy's create_engine, as
a JSON-encoded value
+#
+# Example: sql_alchemy_engine_args = {"arg1": true}
+#
+# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_ENGINE_ARGS
+#
+# sql_alchemy_engine_args =
+
+# The encoding for the databases
+#
+# Variable: AIRFLOW__DATABASE__SQL_ENGINE_ENCODING
+#
+sql_engine_encoding = utf-8
+
+# Collation for ``dag_id``, ``task_id``, ``key``, ``external_executor_id``
columns
+# in case they have different encoding.
+# By default this collation is the same as the database collation, however for
``mysql`` and ``mariadb``
+# the default is ``utf8mb3_bin`` so that the index sizes of our index keys
will not exceed
+# the maximum size of allowed index when collation is set to ``utf8mb4``
variant, see
+# `GitHub Issue Comment
<https://github.com/apache/airflow/pull/17603#issuecomment-901121618>`__
+# for more details.
+#
+# Variable: AIRFLOW__DATABASE__SQL_ENGINE_COLLATION_FOR_IDS
+#
+# sql_engine_collation_for_ids =
+
+# If SQLAlchemy should pool database connections.
+#
+# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_ENABLED
+#
+sql_alchemy_pool_enabled = True
+
+# The SQLAlchemy pool size is the maximum number of database connections
+# in the pool. 0 indicates no limit.
+#
+# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_SIZE
+#
+sql_alchemy_pool_size = 5
+
+# The maximum overflow size of the pool.
+# When the number of checked-out connections reaches the size set in pool_size,
+# additional connections will be returned up to this limit.
+# When those additional connections are returned to the pool, they are
disconnected and discarded.
+# It follows then that the total number of simultaneous connections the pool
will allow
+# is **pool_size** + **max_overflow**,
+# and the total number of "sleeping" connections the pool will allow is
pool_size.
+# max_overflow can be set to ``-1`` to indicate no overflow limit;
+# no limit will be placed on the total number of concurrent connections.
Defaults to ``10``.
+#
+# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_MAX_OVERFLOW
+#
+sql_alchemy_max_overflow = 10
+
+# The SQLAlchemy pool recycle is the number of seconds a connection
+# can be idle in the pool before it is invalidated. This config does
+# not apply to sqlite. If the number of DB connections is ever exceeded,
+# a lower config value will allow the system to recover faster.
+#
+# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_RECYCLE
+#
+sql_alchemy_pool_recycle = 1800
+
+# Check connection at the start of each connection pool checkout.
+# Typically, this is a simple statement like "SELECT 1".
+# See `SQLAlchemy Pooling: Disconnect Handling - Pessimistic
+#
<https://docs.sqlalchemy.org/en/14/core/pooling.html#disconnect-handling-pessimistic>`__
+# for more details.
+#
+# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_PRE_PING
+#
+sql_alchemy_pool_pre_ping = True
+
+# The schema to use for the metadata database.
+# SQLAlchemy supports databases with the concept of multiple schemas.
+#
+# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_SCHEMA
+#
+sql_alchemy_schema =
+
+# Import path for connect args in SQLAlchemy. Defaults to an empty dict.
+# This is useful when you want to configure db engine args that SQLAlchemy
won't parse
+# in connection string. This can be set by passing a dictionary containing the
create engine parameters.
+# For more details about passing create engine parameters (keepalives
variables, timeout etc)
+# in Postgres DB Backend see `Setting up a PostgreSQL Database
+#
<https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html#setting-up-a-postgresql-database>`__
+# e.g ``connect_args={"timeout":30}`` can be defined in
``airflow_local_settings.py`` and
+# can be imported as shown below
+#
+# Example: sql_alchemy_connect_args = airflow_local_settings.connect_args
+#
+# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_CONNECT_ARGS
+#
+# sql_alchemy_connect_args =
+
+# Important Warning: Use of sql_alchemy_session_maker Highly Discouraged
+# Import path for function which returns 'sqlalchemy.orm.sessionmaker'.
+# Improper configuration of sql_alchemy_session_maker can lead to serious
issues,
+# including data corruption, unrecoverable application crashes. Please review
the SQLAlchemy
+# documentation for detailed guidance on proper configuration and best
practices.
+#
+# Example: sql_alchemy_session_maker = airflow_local_settings._sessionmaker
+#
+# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_SESSION_MAKER
+#
+# sql_alchemy_session_maker =
+
+# Whether to load the default connections that ship with Airflow when
``airflow db init`` is called.
+# It's good to get started, but you probably want to set this to ``False`` in
a production environment.
+#
+# Variable: AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS
+#
+load_default_connections = True
+
+# Number of times the code should be retried in case of DB Operational Errors.
+# Not all transactions will be retried as it can cause undesired state.
+# Currently it is only used in ``DagFileProcessor.process_file`` to retry
``dagbag.sync_to_db``.
+#
+# Variable: AIRFLOW__DATABASE__MAX_DB_RETRIES
+#
+max_db_retries = 3
+
+# Whether to run alembic migrations during Airflow start up. Sometimes this
operation can be expensive,
+# and the users can assert the correct version through other means (e.g.
through a Helm chart).
+# Accepts ``True`` or ``False``.
+#
+# Variable: AIRFLOW__DATABASE__CHECK_MIGRATIONS
+#
+check_migrations = True
+
+[logging]
+# The folder where airflow should store its log files.
+# This path must be absolute.
+# There are a few existing configurations that assume this is set to the
default.
+# If you choose to override this you may need to update the
+# ``[logging] dag_processor_manager_log_location`` and
+# ``[logging] child_process_log_directory settings`` as well.
+#
+# Variable: AIRFLOW__LOGGING__BASE_LOG_FOLDER
+#
+base_log_folder = /var/lib/airflow/logs
+
+# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic
Search.
+# Set this to ``True`` if you want to enable remote logging.
+#
+# Variable: AIRFLOW__LOGGING__REMOTE_LOGGING
+#
+remote_logging = False
+
+# Users must supply an Airflow connection id that provides access to the
storage
+# location. Depending on your remote logging service, this may only be used for
+# reading logs, not writing them.
+#
+# Variable: AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID
+#
+remote_log_conn_id =
+
+# Whether the local log files for GCS, S3, WASB and OSS remote logging should
be deleted after
+# they are uploaded to the remote location.
+#
+# Variable: AIRFLOW__LOGGING__DELETE_LOCAL_LOGS
+#
+delete_local_logs = False
+
+# Path to Google Credential JSON file. If omitted, authorization based on `the
Application Default
+# Credentials
+#
<https://cloud.google.com/docs/authentication/application-default-credentials>`__
will
+# be used.
+#
+# Variable: AIRFLOW__LOGGING__GOOGLE_KEY_PATH
+#
+google_key_path =
+
+# Storage bucket URL for remote logging
+# S3 buckets should start with **s3://**
+# Cloudwatch log groups should start with **cloudwatch://**
+# GCS buckets should start with **gs://**
+# WASB buckets should start with **wasb** just to help Airflow select correct
handler
+# Stackdriver logs should start with **stackdriver://**
+#
+# Variable: AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER
+#
+remote_base_log_folder =
+
+# The remote_task_handler_kwargs param is loaded into a dictionary and passed
to the ``__init__``
+# of remote task handler and it overrides the values provided by Airflow
config. For example if you set
+# ``delete_local_logs=False`` and you provide ``{"delete_local_copy": true}``,
then the local
+# log files will be deleted after they are uploaded to remote location.
+#
+# Example: remote_task_handler_kwargs = {"delete_local_copy": true}
+#
+# Variable: AIRFLOW__LOGGING__REMOTE_TASK_HANDLER_KWARGS
+#
+remote_task_handler_kwargs =
+
+# Use server-side encryption for logs stored in S3
+#
+# Variable: AIRFLOW__LOGGING__ENCRYPT_S3_LOGS
+#
+encrypt_s3_logs = False
+
+# Logging level.
+#
+# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``.
+#
+# Variable: AIRFLOW__LOGGING__LOGGING_LEVEL
+#
+logging_level = INFO
+
+# Logging level for celery. If not set, it uses the value of logging_level
+#
+# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``.
+#
+# Variable: AIRFLOW__LOGGING__CELERY_LOGGING_LEVEL
+#
+celery_logging_level =
+
+# Logging level for Flask-appbuilder UI.
+#
+# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``.
+#
+# Variable: AIRFLOW__LOGGING__FAB_LOGGING_LEVEL
+#
+fab_logging_level = WARNING
+
+# Logging class
+# Specify the class that will specify the logging configuration
+# This class has to be on the python classpath
+#
+# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG
+#
+# Variable: AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS
+#
+logging_config_class =
+
+# Flag to enable/disable Colored logs in Console
+# Colour the logs when the controlling terminal is a TTY.
+#
+# Variable: AIRFLOW__LOGGING__COLORED_CONSOLE_LOG
+#
+colored_console_log = True
+
+# Log format for when Colored logs is enabled
+#
+# Variable: AIRFLOW__LOGGING__COLORED_LOG_FORMAT
+#
+colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s]
{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}
%%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s
+
+# Specifies the class utilized by Airflow to implement colored logging
+#
+# Variable: AIRFLOW__LOGGING__COLORED_FORMATTER_CLASS
+#
+colored_formatter_class =
airflow.utils.log.colored_log.CustomTTYColoredFormatter
+
+# Format of Log line
+#
+# Variable: AIRFLOW__LOGGING__LOG_FORMAT
+#
+log_format = [%%(asctime)s] {%%(filename)s:%%(lineno)d} %%(levelname)s -
%%(message)s
+
+# Defines the format of log messages for simple logging configuration
+#
+# Variable: AIRFLOW__LOGGING__SIMPLE_LOG_FORMAT
+#
+simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s
+
+# Where to send dag parser logs. If "file", logs are sent to log files defined
by child_process_log_directory.
+#
+# Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_LOG_TARGET
+#
+dag_processor_log_target = file
+
+# Format of Dag Processor Log line
+#
+# Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_LOG_FORMAT
+#
+dag_processor_log_format = [%%(asctime)s] [SOURCE:DAG_PROCESSOR]
{%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s
+
+# Determines the formatter class used by Airflow for structuring its log
messages
+# The default formatter class is timezone-aware, which means that timestamps
attached to log entries
+# will be adjusted to reflect the local timezone of the Airflow instance
+#
+# Variable: AIRFLOW__LOGGING__LOG_FORMATTER_CLASS
+#
+log_formatter_class = airflow.utils.log.timezone_aware.TimezoneAware
+
+# An import path to a function to add adaptations of each secret added with
+# ``airflow.utils.log.secrets_masker.mask_secret`` to be masked in log
messages. The given function
+# is expected to require a single parameter: the secret to be adapted. It may
return a
+# single adaptation of the secret or an iterable of adaptations to each be
masked as secrets.
+# The original secret will be masked as well as any adaptations returned.
+#
+# Example: secret_mask_adapter = urllib.parse.quote
+#
+# Variable: AIRFLOW__LOGGING__SECRET_MASK_ADAPTER
+#
+secret_mask_adapter =
+
+# Specify prefix pattern like mentioned below with stream handler
``TaskHandlerWithCustomFormatter``
+#
+# Example: task_log_prefix_template =
{{ti.dag_id}}-{{ti.task_id}}-{{execution_date}}-{{ti.try_number}}
+#
+# Variable: AIRFLOW__LOGGING__TASK_LOG_PREFIX_TEMPLATE
+#
+task_log_prefix_template =
+
+# Formatting for how airflow generates file names/paths for each task run.
+#
+# Variable: AIRFLOW__LOGGING__LOG_FILENAME_TEMPLATE
+#
+log_filename_template = dag_id={{ ti.dag_id }}/run_id={{ ti.run_id
}}/task_id={{ ti.task_id }}/{%% if ti.map_index >= 0 %%}map_index={{
ti.map_index }}/{%% endif %%}attempt={{ try_number }}.log
+
+# Formatting for how airflow generates file names for log
+#
+# Variable: AIRFLOW__LOGGING__LOG_PROCESSOR_FILENAME_TEMPLATE
+#
+log_processor_filename_template = {{ filename }}.log
+
+# Full path of dag_processor_manager logfile.
+#
+# Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_LOCATION
+#
+dag_processor_manager_log_location =
/var/lib/airflow/logs/dag_processor_manager/dag_processor_manager.log
+
+# Whether DAG processor manager will write logs to stdout
+#
+# Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_STDOUT
+#
+dag_processor_manager_log_stdout = False
+
+# Name of handler to read task instance logs.
+# Defaults to use ``task`` handler.
+#
+# Variable: AIRFLOW__LOGGING__TASK_LOG_READER
+#
+task_log_reader = task
+
+# A comma\-separated list of third-party logger names that will be configured
to print messages to
+# consoles\.
+#
+# Example: extra_logger_names = connexion,sqlalchemy
+#
+# Variable: AIRFLOW__LOGGING__EXTRA_LOGGER_NAMES
+#
+extra_logger_names =
+
+# When you start an Airflow worker, Airflow starts a tiny web server
+# subprocess to serve the workers local log files to the airflow main
+# web server, who then builds pages and sends them to users. This defines
+# the port on which the logs are served. It needs to be unused, and open
+# visible from the main web server to connect into the workers.
+#
+# Variable: AIRFLOW__LOGGING__WORKER_LOG_SERVER_PORT
+#
+worker_log_server_port = 8793
+
+# Port to serve logs from for triggerer.
+# See ``[logging] worker_log_server_port`` description for more info.
+#
+# Variable: AIRFLOW__LOGGING__TRIGGER_LOG_SERVER_PORT
+#
+trigger_log_server_port = 8794
+
+# We must parse timestamps to interleave logs between trigger and task. To do
so,
+# we need to parse timestamps in log files. In case your log format is
non-standard,
+# you may provide import path to callable which takes a string log line and
returns
+# the timestamp (datetime.datetime compatible).
+#
+# Example: interleave_timestamp_parser = path.to.my_func
+#
+# Variable: AIRFLOW__LOGGING__INTERLEAVE_TIMESTAMP_PARSER
+#
+# interleave_timestamp_parser =
+
+# Permissions in the form or of octal string as understood by chmod. The
permissions are important
+# when you use impersonation, when logs are written by a different user than
airflow. The most secure
+# way of configuring it in this case is to add both users to the same group
and make it the default
+# group of both users. Group-writeable logs are default in airflow, but you
might decide that you are
+# OK with having the logs other-writeable, in which case you should set it to
``0o777``. You might
+# decide to add more security if you do not use impersonation and change it to
``0o755`` to make it
+# only owner-writeable. You can also make it just readable only for owner by
changing it to ``0o700``
+# if all the access (read/write) for your logs happens from the same user.
+#
+# Example: file_task_handler_new_folder_permissions = 0o775
+#
+# Variable: AIRFLOW__LOGGING__FILE_TASK_HANDLER_NEW_FOLDER_PERMISSIONS
+#
+file_task_handler_new_folder_permissions = 0o775
+
+# Permissions in the form or of octal string as understood by chmod. The
permissions are important
+# when you use impersonation, when logs are written by a different user than
airflow. The most secure
+# way of configuring it in this case is to add both users to the same group
and make it the default
+# group of both users. Group-writeable logs are default in airflow, but you
might decide that you are
+# OK with having the logs other-writeable, in which case you should set it to
``0o666``. You might
+# decide to add more security if you do not use impersonation and change it to
``0o644`` to make it
+# only owner-writeable. You can also make it just readable only for owner by
changing it to ``0o600``
+# if all the access (read/write) for your logs happens from the same user.
+#
+# Example: file_task_handler_new_file_permissions = 0o664
+#
+# Variable: AIRFLOW__LOGGING__FILE_TASK_HANDLER_NEW_FILE_PERMISSIONS
+#
+file_task_handler_new_file_permissions = 0o664
+
+# By default Celery sends all logs into stderr.
+# If enabled any previous logging handlers will get *removed*.
+# With this option AirFlow will create new handlers
+# and send low level logs like INFO and WARNING to stdout,
+# while sending higher severity logs to stderr.
+#
+# Variable: AIRFLOW__LOGGING__CELERY_STDOUT_STDERR_SEPARATION
+#
+celery_stdout_stderr_separation = False
+
+# If enabled, Airflow may ship messages to task logs from outside the task run
context, e.g. from
+# the scheduler, executor, or callback execution context. This can help in
circumstances such as
+# when there's something blocking the execution of the task and ordinarily
there may be no task
+# logs at all.
+# This is set to ``True`` by default. If you encounter issues with this feature
+# (e.g. scheduler performance issues) it can be disabled.
+#
+# Variable: AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER
+#
+enable_task_context_logger = True
+
+# A comma separated list of keywords related to errors whose presence should
display the line in red
+# color in UI
+#
+# Variable: AIRFLOW__LOGGING__COLOR_LOG_ERROR_KEYWORDS
+#
+color_log_error_keywords = error,exception
+
+# A comma separated list of keywords related to warning whose presence should
display the line in yellow
+# color in UI
+#
+# Variable: AIRFLOW__LOGGING__COLOR_LOG_WARNING_KEYWORDS
+#
+color_log_warning_keywords = warn
+
+[metrics]
+# `StatsD <https://github.com/statsd/statsd>`__ integration settings.
+
+# If true, ``[metrics] metrics_allow_list`` and ``[metrics]
metrics_block_list`` will use
+# regex pattern matching anywhere within the metric name instead of only
prefix matching
+# at the start of the name.
+#
+# Variable: AIRFLOW__METRICS__METRICS_USE_PATTERN_MATCH
+#
+metrics_use_pattern_match = False
+
+# Configure an allow list (comma separated string) to send only certain
metrics.
+# If ``[metrics] metrics_use_pattern_match`` is ``false``, match only the
exact metric name prefix.
+# If ``[metrics] metrics_use_pattern_match`` is ``true``, provide regex
patterns to match.
+#
+# Example: metrics_allow_list =
"scheduler,executor,dagrun,pool,triggerer,celery" or
"^scheduler,^executor,heartbeat|timeout"
+#
+# Variable: AIRFLOW__METRICS__METRICS_ALLOW_LIST
+#
+metrics_allow_list =
+
+# Configure a block list (comma separated string) to block certain metrics
from being emitted.
+# If ``[metrics] metrics_allow_list`` and ``[metrics] metrics_block_list`` are
both configured,
+# ``[metrics] metrics_block_list`` is ignored.
+#
+# If ``[metrics] metrics_use_pattern_match`` is ``false``, match only the
exact metric name prefix.
+#
+# If ``[metrics] metrics_use_pattern_match`` is ``true``, provide regex
patterns to match.
+#
+# Example: metrics_block_list =
"scheduler,executor,dagrun,pool,triggerer,celery" or
"^scheduler,^executor,heartbeat|timeout"
+#
+# Variable: AIRFLOW__METRICS__METRICS_BLOCK_LIST
+#
+metrics_block_list =
+
+# Enables sending metrics to StatsD.
+#
+# Variable: AIRFLOW__METRICS__STATSD_ON
+#
+statsd_on = False
+
+# Specifies the host address where the StatsD daemon (or server) is running
+#
+# Variable: AIRFLOW__METRICS__STATSD_HOST
+#
+statsd_host = localhost
+
+# Specifies the port on which the StatsD daemon (or server) is listening to
+#
+# Variable: AIRFLOW__METRICS__STATSD_PORT
+#
+statsd_port = 8125
+
+# Defines the namespace for all metrics sent from Airflow to StatsD
+#
+# Variable: AIRFLOW__METRICS__STATSD_PREFIX
+#
+statsd_prefix = airflow
+
+# A function that validate the StatsD stat name, apply changes to the stat
name if necessary and return
+# the transformed stat name.
+#
+# The function should have the following signature
+#
+# .. code-block:: python
+#
+# def func_name(stat_name: str) -> str: ...
+#
+# Variable: AIRFLOW__METRICS__STAT_NAME_HANDLER
+#
+stat_name_handler =
+
+# To enable datadog integration to send airflow metrics.
+#
+# Variable: AIRFLOW__METRICS__STATSD_DATADOG_ENABLED
+#
+statsd_datadog_enabled = False
+
+# List of datadog tags attached to all metrics(e.g:
``key1:value1,key2:value2``)
+#
+# Variable: AIRFLOW__METRICS__STATSD_DATADOG_TAGS
+#
+statsd_datadog_tags =
+
+# Set to ``False`` to disable metadata tags for some of the emitted metrics
+#
+# Variable: AIRFLOW__METRICS__STATSD_DATADOG_METRICS_TAGS
+#
+statsd_datadog_metrics_tags = True
+
+# If you want to utilise your own custom StatsD client set the relevant
+# module path below.
+# Note: The module path must exist on your
+# `PYTHONPATH <https://docs.python.org/3/using/cmdline.html#envvar-PYTHONPATH>`
+# for Airflow to pick it up
+#
+# Variable: AIRFLOW__METRICS__STATSD_CUSTOM_CLIENT_PATH
+#
+# statsd_custom_client_path =
+
+# If you want to avoid sending all the available metrics tags to StatsD,
+# you can configure a block list of prefixes (comma separated) to filter out
metric tags
+# that start with the elements of the list (e.g: ``job_id,run_id``)
+#
+# Example: statsd_disabled_tags = job_id,run_id,dag_id,task_id
+#
+# Variable: AIRFLOW__METRICS__STATSD_DISABLED_TAGS
+#
+statsd_disabled_tags = job_id,run_id
+
+# To enable sending Airflow metrics with StatsD-Influxdb tagging convention.
+#
+# Variable: AIRFLOW__METRICS__STATSD_INFLUXDB_ENABLED
+#
+statsd_influxdb_enabled = False
+
+# Enables sending metrics to OpenTelemetry.
+#
+# Variable: AIRFLOW__METRICS__OTEL_ON
+#
+otel_on = False
+
+# Specifies the hostname or IP address of the OpenTelemetry Collector to which
Airflow sends
+# metrics and traces.
+#
+# Variable: AIRFLOW__METRICS__OTEL_HOST
+#
+otel_host = localhost
+
+# Specifies the port of the OpenTelemetry Collector that is listening to.
+#
+# Variable: AIRFLOW__METRICS__OTEL_PORT
+#
+otel_port = 8889
+
+# The prefix for the Airflow metrics.
+#
+# Variable: AIRFLOW__METRICS__OTEL_PREFIX
+#
+otel_prefix = airflow
+
+# Defines the interval, in milliseconds, at which Airflow sends batches of
metrics and traces
+# to the configured OpenTelemetry Collector.
+#
+# Variable: AIRFLOW__METRICS__OTEL_INTERVAL_MILLISECONDS
+#
+otel_interval_milliseconds = 60000
+
+# If ``True``, all metrics are also emitted to the console. Defaults to
``False``.
+#
+# Variable: AIRFLOW__METRICS__OTEL_DEBUGGING_ON
+#
+otel_debugging_on = False
+
+# The default service name of traces.
+#
+# Variable: AIRFLOW__METRICS__OTEL_SERVICE
+#
+otel_service = Airflow
+
+# If ``True``, SSL will be enabled. Defaults to ``False``.
+# To establish an HTTPS connection to the OpenTelemetry collector,
+# you need to configure the SSL certificate and key within the OpenTelemetry
collector's
+# ``config.yml`` file.
+#
+# Variable: AIRFLOW__METRICS__OTEL_SSL_ACTIVE
+#
+otel_ssl_active = False
+
+[traces]
+# Distributed traces integration settings.
+
+# Enables sending traces to OpenTelemetry.
+#
+# Variable: AIRFLOW__TRACES__OTEL_ON
+#
+otel_on = False
+
+# Specifies the hostname or IP address of the OpenTelemetry Collector to which
Airflow sends
+# traces.
+#
+# Variable: AIRFLOW__TRACES__OTEL_HOST
+#
+otel_host = localhost
+
+# Specifies the port of the OpenTelemetry Collector that is listening to.
+#
+# Variable: AIRFLOW__TRACES__OTEL_PORT
+#
+otel_port = 8889
+
+# The default service name of traces.
+#
+# Variable: AIRFLOW__TRACES__OTEL_SERVICE
+#
+otel_service = Airflow
+
+# If True, all traces are also emitted to the console. Defaults to False.
+#
+# Variable: AIRFLOW__TRACES__OTEL_DEBUGGING_ON
+#
+otel_debugging_on = False
+
+# If True, SSL will be enabled. Defaults to False.
+# To establish an HTTPS connection to the OpenTelemetry collector,
+# you need to configure the SSL certificate and key within the OpenTelemetry
collector's
+# config.yml file.
+#
+# Variable: AIRFLOW__TRACES__OTEL_SSL_ACTIVE
+#
+otel_ssl_active = False
+
+# If True, after the task is complete, the full task log messages will be
added as the
+# span events, chunked by 64k size. defaults to False.
+#
+# Variable: AIRFLOW__TRACES__OTEL_TASK_LOG_EVENT
+#
+otel_task_log_event = False
+
+[secrets]
+# Full class name of secrets backend to enable (will precede env vars and
metastore in search path)
+#
+# Example: backend =
airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend
+#
+# Variable: AIRFLOW__SECRETS__BACKEND
+#
+backend =
+
+# The backend_kwargs param is loaded into a dictionary and passed to
``__init__``
+# of secrets backend class. See documentation for the secrets backend you are
using.
+# JSON is expected.
+#
+# Example for AWS Systems Manager ParameterStore:
+# ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}``
+#
+# Variable: AIRFLOW__SECRETS__BACKEND_KWARGS
+#
+backend_kwargs =
+
+# .. note:: |experimental|
+#
+# Enables local caching of Variables, when parsing DAGs only.
+# Using this option can make dag parsing faster if Variables are used in top
level code, at the expense
+# of longer propagation time for changes.
+# Please note that this cache concerns only the DAG parsing step. There is no
caching in place when DAG
+# tasks are run.
+#
+# Variable: AIRFLOW__SECRETS__USE_CACHE
+#
+use_cache = False
+
+# .. note:: |experimental|
+#
+# When the cache is enabled, this is the duration for which we consider an
entry in the cache to be
+# valid. Entries are refreshed if they are older than this many seconds.
+# It means that when the cache is enabled, this is the maximum amount of time
you need to wait to see a
+# Variable change take effect.
+#
+# Variable: AIRFLOW__SECRETS__CACHE_TTL_SECONDS
+#
+cache_ttl_seconds = 900
+
+[cli]
+# In what way should the cli access the API. The LocalClient will use the
+# database directly, while the json_client will use the api running on the
+# webserver
+#
+# Variable: AIRFLOW__CLI__API_CLIENT
+#
+api_client = airflow.api.client.local_client
+
+# If you set web_server_url_prefix, do NOT forget to append it here, ex:
+# ``endpoint_url = http://localhost:8080/myroot``
+# So api will look like: ``http://localhost:8080/myroot/api/experimental/...``
+#
+# Variable: AIRFLOW__CLI__ENDPOINT_URL
+#
+endpoint_url = http://localhost:8080
+
+[debug]
+# Used only with ``DebugExecutor``. If set to ``True`` DAG will fail with first
+# failed task. Helpful for debugging purposes.
+#
+# Variable: AIRFLOW__DEBUG__FAIL_FAST
+#
+fail_fast = False
+
+[api]
+# Enables the deprecated experimental API. Please note that these API
endpoints do not have
+# access control. An authenticated user has full access.
+#
+# .. warning::
+#
+# This `Experimental REST API
+#
<https://airflow.apache.org/docs/apache-airflow/stable/deprecated-rest-api-ref.html>`__
is
+# deprecated since version 2.0. Please consider using
+# `the Stable REST API
+#
<https://airflow.apache.org/docs/apache-airflow/stable/stable-rest-api-ref.html>`__.
+# For more information on migration, see
+# `RELEASE_NOTES.rst
<https://github.com/apache/airflow/blob/main/RELEASE_NOTES.rst>`_
+#
+# Variable: AIRFLOW__API__ENABLE_EXPERIMENTAL_API
+#
+enable_experimental_api = False
+
+# Comma separated list of auth backends to authenticate users of the API. See
+# `Security: API
+# <https://airflow.apache.org/docs/apache-airflow/stable/security/api.html>`__
for possible values.
+# ("airflow.api.auth.backend.default" allows all requests for historic reasons)
+#
+# Variable: AIRFLOW__API__AUTH_BACKENDS
+#
+auth_backends = airflow.api.auth.backend.session
+
+# Used to set the maximum page limit for API requests. If limit passed as param
+# is greater than maximum page limit, it will be ignored and maximum page
limit value
+# will be set as the limit
+#
+# Variable: AIRFLOW__API__MAXIMUM_PAGE_LIMIT
+#
+maximum_page_limit = 100
+
+# Used to set the default page limit when limit param is zero or not provided
in API
+# requests. Otherwise if positive integer is passed in the API requests as
limit, the
+# smallest number of user given limit or maximum page limit is taken as limit.
+#
+# Variable: AIRFLOW__API__FALLBACK_PAGE_LIMIT
+#
+fallback_page_limit = 100
+
+# The intended audience for JWT token credentials used for authorization. This
value must match on the client and server sides. If empty, audience will not be
tested.
+#
+# Example: google_oauth2_audience =
project-id-random-value.apps.googleusercontent.com
+#
+# Variable: AIRFLOW__API__GOOGLE_OAUTH2_AUDIENCE
+#
+google_oauth2_audience =
+
+# Path to Google Cloud Service Account key file (JSON). If omitted,
authorization based on
+# `the Application Default Credentials
+#
<https://cloud.google.com/docs/authentication/production#finding_credentials_automatically>`__
will
+# be used.
+#
+# Example: google_key_path = /files/service-account-json
+#
+# Variable: AIRFLOW__API__GOOGLE_KEY_PATH
+#
+google_key_path =
+
+# Used in response to a preflight request to indicate which HTTP
+# headers can be used when making the actual request. This header is
+# the server side response to the browser's
+# Access-Control-Request-Headers header.
+#
+# Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_HEADERS
+#
+access_control_allow_headers =
+
+# Specifies the method or methods allowed when accessing the resource.
+#
+# Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_METHODS
+#
+access_control_allow_methods =
+
+# Indicates whether the response can be shared with requesting code from the
given origins.
+# Separate URLs with space.
+#
+# Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_ORIGINS
+#
+access_control_allow_origins =
+
+# Indicates whether the **xcomEntries** endpoint supports the **deserialize**
+# flag. If set to ``False``, setting this flag in a request would result in a
+# 400 Bad Request error.
+#
+# Variable: AIRFLOW__API__ENABLE_XCOM_DESERIALIZE_SUPPORT
+#
+enable_xcom_deserialize_support = False
+
+[lineage]
+# what lineage backend to use
+#
+# Variable: AIRFLOW__LINEAGE__BACKEND
+#
+backend =
+
+[operators]
+# The default owner assigned to each new operator, unless
+# provided explicitly or passed via ``default_args``
+#
+# Variable: AIRFLOW__OPERATORS__DEFAULT_OWNER
+#
+default_owner = airflow
+
+# The default value of attribute "deferrable" in operators and sensors.
+#
+# Variable: AIRFLOW__OPERATORS__DEFAULT_DEFERRABLE
+#
+default_deferrable = false
+
+# Indicates the default number of CPU units allocated to each operator when no
specific CPU request
+# is specified in the operator's configuration
+#
+# Variable: AIRFLOW__OPERATORS__DEFAULT_CPUS
+#
+default_cpus = 1
+
+# Indicates the default number of RAM allocated to each operator when no
specific RAM request
+# is specified in the operator's configuration
+#
+# Variable: AIRFLOW__OPERATORS__DEFAULT_RAM
+#
+default_ram = 512
+
+# Indicates the default number of disk storage allocated to each operator when
no specific disk request
+# is specified in the operator's configuration
+#
+# Variable: AIRFLOW__OPERATORS__DEFAULT_DISK
+#
+default_disk = 512
+
+# Indicates the default number of GPUs allocated to each operator when no
specific GPUs request
+# is specified in the operator's configuration
+#
+# Variable: AIRFLOW__OPERATORS__DEFAULT_GPUS
+#
+default_gpus = 0
+
+# Default queue that tasks get assigned to and that worker listen on.
+#
+# Variable: AIRFLOW__OPERATORS__DEFAULT_QUEUE
+#
+default_queue = default
+
+# Is allowed to pass additional/unused arguments (args, kwargs) to the
BaseOperator operator.
+# If set to ``False``, an exception will be thrown,
+# otherwise only the console message will be displayed.
+#
+# Variable: AIRFLOW__OPERATORS__ALLOW_ILLEGAL_ARGUMENTS
+#
+allow_illegal_arguments = False
+
+[webserver]
+# The message displayed when a user attempts to execute actions beyond their
authorised privileges.
+#
+# Variable: AIRFLOW__WEBSERVER__ACCESS_DENIED_MESSAGE
+#
+access_denied_message = Access is Denied
+
+# Path of webserver config file used for configuring the webserver parameters
+#
+# Variable: AIRFLOW__WEBSERVER__CONFIG_FILE
+#
+config_file = /var/lib/airflow/webserver_config.py
+
+# The base url of your website: Airflow cannot guess what domain or CNAME you
are using.
+# This is used to create links in the Log Url column in the Browse - Task
Instances menu,
+# as well as in any automated emails sent by Airflow that contain links to
your webserver.
+#
+# Variable: AIRFLOW__WEBSERVER__BASE_URL
+#
+base_url = http://localhost:8080
+
+# Default timezone to display all dates in the UI, can be UTC, system, or
+# any IANA timezone string (e.g. **Europe/Amsterdam**). If left empty the
+# default value of core/default_timezone will be used
+#
+# Example: default_ui_timezone = America/New_York
+#
+# Variable: AIRFLOW__WEBSERVER__DEFAULT_UI_TIMEZONE
+#
+default_ui_timezone = UTC
+
+# The ip specified when starting the web server
+#
+# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_HOST
+#
+web_server_host = 0.0.0.0
+
+# The port on which to run the web server
+#
+# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_PORT
+#
+web_server_port = 8080
+
+# Paths to the SSL certificate and key for the web server. When both are
+# provided SSL will be enabled. This does not change the web server port.
+#
+# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_SSL_CERT
+#
+web_server_ssl_cert =
+
+# Paths to the SSL certificate and key for the web server. When both are
+# provided SSL will be enabled. This does not change the web server port.
+#
+# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY
+#
+web_server_ssl_key =
+
+# The type of backend used to store web session data, can be ``database`` or
``securecookie``. For the
+# ``database`` backend, sessions are store in the database and they can be
+# managed there (for example when you reset password of the user, all sessions
for that user are
+# deleted). For the ``securecookie`` backend, sessions are stored in encrypted
cookies on the client
+# side. The ``securecookie`` mechanism is 'lighter' than database backend, but
sessions are not deleted
+# when you reset password of the user, which means that other than waiting for
expiry time, the only
+# way to invalidate all sessions for a user is to change secret_key and
restart webserver (which
+# also invalidates and logs out all other user's sessions).
+#
+# When you are using ``database`` backend, make sure to keep your database
session table small
+# by periodically running ``airflow db clean --table session`` command,
especially if you have
+# automated API calls that will create a new session for each call rather than
reuse the sessions
+# stored in browser cookies.
+#
+# Example: session_backend = securecookie
+#
+# Variable: AIRFLOW__WEBSERVER__SESSION_BACKEND
+#
+session_backend = database
+
+# Number of seconds the webserver waits before killing gunicorn master that
doesn't respond
+#
+# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_MASTER_TIMEOUT
+#
+web_server_master_timeout = 120
+
+# Number of seconds the gunicorn webserver waits before timing out on a worker
+#
+# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_WORKER_TIMEOUT
+#
+web_server_worker_timeout = 120
+
+# Number of workers to refresh at a time. When set to 0, worker refresh is
+# disabled. When nonzero, airflow periodically refreshes webserver workers by
+# bringing up new ones and killing old ones.
+#
+# Variable: AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE
+#
+worker_refresh_batch_size = 1
+
+# Number of seconds to wait before refreshing a batch of workers.
+#
+# Variable: AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL
+#
+worker_refresh_interval = 6000
+
+# If set to ``True``, Airflow will track files in plugins_folder directory.
When it detects changes,
+# then reload the gunicorn. If set to ``True``, gunicorn starts without
preloading, which is slower,
+# uses more memory, and may cause race conditions. Avoid setting this to
``True`` in production.
+#
+# Variable: AIRFLOW__WEBSERVER__RELOAD_ON_PLUGIN_CHANGE
+#
+reload_on_plugin_change = False
+
+# Secret key used to run your flask app. It should be as random as possible.
However, when running
+# more than 1 instances of webserver, make sure all of them use the same
``secret_key`` otherwise
+# one of them will error with "CSRF session token is missing".
+# The webserver key is also used to authorize requests to Celery workers when
logs are retrieved.
+# The token generated using the secret key has a short expiry time though -
make sure that time on
+# ALL the machines that you run airflow components on is synchronized (for
example using ntpd)
+# otherwise you might get "forbidden" errors when the logs are accessed.
+#
+# Variable: AIRFLOW__WEBSERVER__SECRET_KEY
+#
+secret_key = dAtBycpTM5FZ3iBrvRjUGA==
+
+# Number of workers to run the Gunicorn web server
+#
+# Variable: AIRFLOW__WEBSERVER__WORKERS
+#
+workers = 4
+
+# The worker class gunicorn should use. Choices include
+# ``sync`` (default), ``eventlet``, ``gevent``.
+#
+# .. warning::
+#
+# When using ``gevent`` you might also want to set the
``_AIRFLOW_PATCH_GEVENT``
+# environment variable to ``"1"`` to make sure gevent patching is done as
early as possible.
+#
+# Be careful to set ``_AIRFLOW_PATCH_GEVENT`` only on the web server as
gevent patching may
+# affect the scheduler behavior via the ``multiprocessing`` sockets module
and cause crash.
+#
+# See related Issues / PRs for more details:
+#
+# * https://github.com/benoitc/gunicorn/issues/2796
+# * https://github.com/apache/airflow/issues/8212
+# * https://github.com/apache/airflow/pull/28283
+#
+# Variable: AIRFLOW__WEBSERVER__WORKER_CLASS
+#
+worker_class = sync
+
+# Log files for the gunicorn webserver. '-' means log to stderr.
+#
+# Variable: AIRFLOW__WEBSERVER__ACCESS_LOGFILE
+#
+access_logfile = -
+
+# Log files for the gunicorn webserver. '-' means log to stderr.
+#
+# Variable: AIRFLOW__WEBSERVER__ERROR_LOGFILE
+#
+error_logfile = -
+
+# Access log format for gunicorn webserver.
+# default format is ``%%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s
"%%(f)s" "%%(a)s"``
+# See `Gunicorn Settings: 'access_log_format' Reference
+# <https://docs.gunicorn.org/en/stable/settings.html#access-log-format>`__ for
more details
+#
+# Variable: AIRFLOW__WEBSERVER__ACCESS_LOGFORMAT
+#
+access_logformat =
+
+# Expose the configuration file in the web server. Set to
``non-sensitive-only`` to show all values
+# except those that have security implications. ``True`` shows all values.
``False`` hides the
+# configuration completely.
+#
+# Variable: AIRFLOW__WEBSERVER__EXPOSE_CONFIG
+#
+expose_config = False
+
+# Expose hostname in the web server
+#
+# Variable: AIRFLOW__WEBSERVER__EXPOSE_HOSTNAME
+#
+expose_hostname = False
+
+# Expose stacktrace in the web server
+#
+# Variable: AIRFLOW__WEBSERVER__EXPOSE_STACKTRACE
+#
+expose_stacktrace = False
+
+# Default DAG view. Valid values are: ``grid``, ``graph``, ``duration``,
``gantt``, ``landing_times``
+#
+# Variable: AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW
+#
+dag_default_view = grid
+
+# Default DAG orientation. Valid values are:
+# ``LR`` (Left->Right), ``TB`` (Top->Bottom), ``RL`` (Right->Left), ``BT``
(Bottom->Top)
+#
+# Variable: AIRFLOW__WEBSERVER__DAG_ORIENTATION
+#
+dag_orientation = LR
+
+# Sorting order in grid view. Valid values are: ``topological``,
``hierarchical_alphabetical``
+#
+# Variable: AIRFLOW__WEBSERVER__GRID_VIEW_SORTING_ORDER
+#
+grid_view_sorting_order = topological
+
+# The amount of time (in secs) webserver will wait for initial handshake
+# while fetching logs from other worker machine
+#
+# Variable: AIRFLOW__WEBSERVER__LOG_FETCH_TIMEOUT_SEC
+#
+log_fetch_timeout_sec = 5
+
+# Time interval (in secs) to wait before next log fetching.
+#
+# Variable: AIRFLOW__WEBSERVER__LOG_FETCH_DELAY_SEC
+#
+log_fetch_delay_sec = 2
+
+# Distance away from page bottom to enable auto tailing.
+#
+# Variable: AIRFLOW__WEBSERVER__LOG_AUTO_TAILING_OFFSET
+#
+log_auto_tailing_offset = 30
+
+# Animation speed for auto tailing log display.
+#
+# Variable: AIRFLOW__WEBSERVER__LOG_ANIMATION_SPEED
+#
+log_animation_speed = 1000
+
+# By default, the webserver shows paused DAGs. Flip this to hide paused
+# DAGs by default
+#
+# Variable: AIRFLOW__WEBSERVER__HIDE_PAUSED_DAGS_BY_DEFAULT
+#
+hide_paused_dags_by_default = False
+
+# Consistent page size across all listing views in the UI
+#
+# Variable: AIRFLOW__WEBSERVER__PAGE_SIZE
+#
+page_size = 100
+
+# Define the color of navigation bar
+#
+# Variable: AIRFLOW__WEBSERVER__NAVBAR_COLOR
+#
+navbar_color = #fff
+
+# Define the color of text in the navigation bar
+#
+# Variable: AIRFLOW__WEBSERVER__NAVBAR_TEXT_COLOR
+#
+navbar_text_color = #51504f
+
+# Define the color of navigation bar links when hovered
+#
+# Variable: AIRFLOW__WEBSERVER__NAVBAR_HOVER_COLOR
+#
+navbar_hover_color = #eee
+
+# Define the color of text in the navigation bar when hovered
+#
+# Variable: AIRFLOW__WEBSERVER__NAVBAR_TEXT_HOVER_COLOR
+#
+navbar_text_hover_color = #51504f
+
+# Define the color of the logo text
+#
+# Variable: AIRFLOW__WEBSERVER__NAVBAR_LOGO_TEXT_COLOR
+#
+navbar_logo_text_color = #51504f
+
+# Default dagrun to show in UI
+#
+# Variable: AIRFLOW__WEBSERVER__DEFAULT_DAG_RUN_DISPLAY_NUMBER
+#
+default_dag_run_display_number = 25
+
+# Enable werkzeug ``ProxyFix`` middleware for reverse proxy
+#
+# Variable: AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX
+#
+enable_proxy_fix = False
+
+# Number of values to trust for ``X-Forwarded-For``.
+# See `Werkzeug: X-Forwarded-For Proxy Fix
+# <https://werkzeug.palletsprojects.com/en/2.3.x/middleware/proxy_fix/>`__ for
more details.
+#
+# Variable: AIRFLOW__WEBSERVER__PROXY_FIX_X_FOR
+#
+proxy_fix_x_for = 1
+
+# Number of values to trust for ``X-Forwarded-Proto``.
+# See `Werkzeug: X-Forwarded-For Proxy Fix
+# <https://werkzeug.palletsprojects.com/en/2.3.x/middleware/proxy_fix/>`__ for
more details.
+#
+# Variable: AIRFLOW__WEBSERVER__PROXY_FIX_X_PROTO
+#
+proxy_fix_x_proto = 1
+
+# Number of values to trust for ``X-Forwarded-Host``.
+# See `Werkzeug: X-Forwarded-For Proxy Fix
+# <https://werkzeug.palletsprojects.com/en/2.3.x/middleware/proxy_fix/>`__ for
more details.
+#
+# Variable: AIRFLOW__WEBSERVER__PROXY_FIX_X_HOST
+#
+proxy_fix_x_host = 1
+
+# Number of values to trust for ``X-Forwarded-Port``.
+# See `Werkzeug: X-Forwarded-For Proxy Fix
+# <https://werkzeug.palletsprojects.com/en/2.3.x/middleware/proxy_fix/>`__ for
more details.
+#
+# Variable: AIRFLOW__WEBSERVER__PROXY_FIX_X_PORT
+#
+proxy_fix_x_port = 1
+
+# Number of values to trust for ``X-Forwarded-Prefix``.
+# See `Werkzeug: X-Forwarded-For Proxy Fix
+# <https://werkzeug.palletsprojects.com/en/2.3.x/middleware/proxy_fix/>`__ for
more details.
+#
+# Variable: AIRFLOW__WEBSERVER__PROXY_FIX_X_PREFIX
+#
+proxy_fix_x_prefix = 1
+
+# Set secure flag on session cookie
+#
+# Variable: AIRFLOW__WEBSERVER__COOKIE_SECURE
+#
+cookie_secure = False
+
+# Set samesite policy on session cookie
+#
+# Variable: AIRFLOW__WEBSERVER__COOKIE_SAMESITE
+#
+cookie_samesite = Lax
+
+# Default setting for wrap toggle on DAG code and TI log views.
+#
+# Variable: AIRFLOW__WEBSERVER__DEFAULT_WRAP
+#
+default_wrap = False
+
+# Allow the UI to be rendered in a frame
+#
+# Variable: AIRFLOW__WEBSERVER__X_FRAME_ENABLED
+#
+x_frame_enabled = True
+
+# Send anonymous user activity to your analytics tool
+# choose from ``google_analytics``, ``segment``, ``metarouter``, or ``matomo``
+#
+# Variable: AIRFLOW__WEBSERVER__ANALYTICS_TOOL
+#
+# analytics_tool =
+
+# Unique ID of your account in the analytics tool
+#
+# Variable: AIRFLOW__WEBSERVER__ANALYTICS_ID
+#
+# analytics_id =
+
+# Your instances url, only applicable to Matomo.
+#
+# Example: analytics_url = https://your.matomo.instance.com/
+#
+# Variable: AIRFLOW__WEBSERVER__ANALYTICS_URL
+#
+# analytics_url =
+
+# 'Recent Tasks' stats will show for old DagRuns if set
+#
+# Variable: AIRFLOW__WEBSERVER__SHOW_RECENT_STATS_FOR_COMPLETED_RUNS
+#
+show_recent_stats_for_completed_runs = True
+
+# The UI cookie lifetime in minutes. User will be logged out from UI after
+# ``[webserver] session_lifetime_minutes`` of non-activity
+#
+# Variable: AIRFLOW__WEBSERVER__SESSION_LIFETIME_MINUTES
+#
+session_lifetime_minutes = 43200
+
+# Sets a custom page title for the DAGs overview page and site title for all
pages
+#
+# Variable: AIRFLOW__WEBSERVER__INSTANCE_NAME
+#
+# instance_name =
+
+# Whether the custom page title for the DAGs overview page contains any Markup
language
+#
+# Variable: AIRFLOW__WEBSERVER__INSTANCE_NAME_HAS_MARKUP
+#
+instance_name_has_markup = False
+
+# How frequently, in seconds, the DAG data will auto-refresh in graph or grid
view
+# when auto-refresh is turned on
+#
+# Variable: AIRFLOW__WEBSERVER__AUTO_REFRESH_INTERVAL
+#
+auto_refresh_interval = 3
+
+# Boolean for displaying warning for publicly viewable deployment
+#
+# Variable: AIRFLOW__WEBSERVER__WARN_DEPLOYMENT_EXPOSURE
+#
+warn_deployment_exposure = True
+
+# Comma separated string of view events to exclude from dag audit view.
+# All other events will be added minus the ones passed here.
+# The audit logs in the db will not be affected by this parameter.
+#
+# Example: audit_view_excluded_events = cli_task_run,running,success
+#
+# Variable: AIRFLOW__WEBSERVER__AUDIT_VIEW_EXCLUDED_EVENTS
+#
+# audit_view_excluded_events =
+
+# Comma separated string of view events to include in dag audit view.
+# If passed, only these events will populate the dag audit view.
+# The audit logs in the db will not be affected by this parameter.
+#
+# Example: audit_view_included_events = dagrun_cleared,failed
+#
+# Variable: AIRFLOW__WEBSERVER__AUDIT_VIEW_INCLUDED_EVENTS
+#
+# audit_view_included_events =
+
+# Boolean for running SwaggerUI in the webserver.
+#
+# Variable: AIRFLOW__WEBSERVER__ENABLE_SWAGGER_UI
+#
+enable_swagger_ui = True
+
+# Boolean for running Internal API in the webserver.
+#
+# Variable: AIRFLOW__WEBSERVER__RUN_INTERNAL_API
+#
+run_internal_api = False
+
+# The caching algorithm used by the webserver. Must be a valid hashlib
function name.
+#
+# Example: caching_hash_method = sha256
+#
+# Variable: AIRFLOW__WEBSERVER__CACHING_HASH_METHOD
+#
+caching_hash_method = md5
+
+# Behavior of the trigger DAG run button for DAGs without params. ``False`` to
skip and trigger
+# without displaying a form to add a **dag_run.conf**, ``True`` to always
display the form.
+# The form is displayed always if parameters are defined.
+#
+# Variable: AIRFLOW__WEBSERVER__SHOW_TRIGGER_FORM_IF_NO_PARAMS
+#
+show_trigger_form_if_no_params = False
+
+# Number of recent DAG run configurations in the selector on the trigger web
form.
+#
+# Example: num_recent_configurations_for_trigger = 10
+#
+# Variable: AIRFLOW__WEBSERVER__NUM_RECENT_CONFIGURATIONS_FOR_TRIGGER
+#
+num_recent_configurations_for_trigger = 5
+
+# A DAG author is able to provide any raw HTML into ``doc_md`` or params
description in
+# ``description_md`` for text formatting. This is including potentially unsafe
javascript.
+# Displaying the DAG or trigger form in web UI provides the DAG author the
potential to
+# inject malicious code into clients browsers. To ensure the web UI is safe by
default,
+# raw HTML is disabled by default. If you trust your DAG authors, you can
enable HTML
+# support in markdown by setting this option to ``True``.
+#
+# This parameter also enables the deprecated fields ``description_html`` and
+# ``custom_html_form`` in DAG params until the feature is removed in a future
version.
+#
+# Example: allow_raw_html_descriptions = False
+#
+# Variable: AIRFLOW__WEBSERVER__ALLOW_RAW_HTML_DESCRIPTIONS
+#
+allow_raw_html_descriptions = False
+
+# The maximum size of the request payload (in MB) that can be sent.
+#
+# Variable: AIRFLOW__WEBSERVER__ALLOWED_PAYLOAD_SIZE
+#
+allowed_payload_size = 1.0
+
+# Require confirmation when changing a DAG in the web UI. This is to prevent
accidental changes
+# to a DAG that may be running on sensitive environments like production.
+# When set to ``True``, confirmation dialog will be shown when a user tries to
Pause/Unpause,
+# Trigger a DAG
+#
+# Variable: AIRFLOW__WEBSERVER__REQUIRE_CONFIRMATION_DAG_CHANGE
+#
+require_confirmation_dag_change = False
+
+[email]
+# Configuration email backend and whether to
+# send email alerts on retry or failure
+
+# Email backend to use
+#
+# Variable: AIRFLOW__EMAIL__EMAIL_BACKEND
+#
+email_backend = airflow.utils.email.send_email_smtp
+
+# Email connection to use
+#
+# Variable: AIRFLOW__EMAIL__EMAIL_CONN_ID
+#
+email_conn_id = smtp_default
+
+# Whether email alerts should be sent when a task is retried
+#
+# Variable: AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_RETRY
+#
+default_email_on_retry = True
+
+# Whether email alerts should be sent when a task failed
+#
+# Variable: AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_FAILURE
+#
+default_email_on_failure = True
+
+# File that will be used as the template for Email subject (which will be
rendered using Jinja2).
+# If not set, Airflow uses a base template.
+#
+# Example: subject_template = /path/to/my_subject_template_file
+#
+# Variable: AIRFLOW__EMAIL__SUBJECT_TEMPLATE
+#
+# subject_template =
+
+# File that will be used as the template for Email content (which will be
rendered using Jinja2).
+# If not set, Airflow uses a base template.
+#
+# Example: html_content_template = /path/to/my_html_content_template_file
+#
+# Variable: AIRFLOW__EMAIL__HTML_CONTENT_TEMPLATE
+#
+# html_content_template =
+
+# Email address that will be used as sender address.
+# It can either be raw email or the complete address in a format ``Sender Name
<sen...@email.com>``
+#
+# Example: from_email = Airflow <airf...@example.com>
+#
+# Variable: AIRFLOW__EMAIL__FROM_EMAIL
+#
+# from_email =
+
+# ssl context to use when using SMTP and IMAP SSL connections. By default, the
context is "default"
+# which sets it to ``ssl.create_default_context()`` which provides the right
balance between
+# compatibility and security, it however requires that certificates in your
operating system are
+# updated and that SMTP/IMAP servers of yours have valid certificates that
have corresponding public
+# keys installed on your machines. You can switch it to "none" if you want to
disable checking
+# of the certificates, but it is not recommended as it allows MITM
(man-in-the-middle) attacks
+# if your infrastructure is not sufficiently secured. It should only be set
temporarily while you
+# are fixing your certificate configuration. This can be typically done by
upgrading to newer
+# version of the operating system you run Airflow components on,by
upgrading/refreshing proper
+# certificates in the OS or by updating certificates for your mail servers.
+#
+# Example: ssl_context = default
+#
+# Variable: AIRFLOW__EMAIL__SSL_CONTEXT
+#
+ssl_context = default
+
+[smtp]
+# If you want airflow to send emails on retries, failure, and you want to use
+# the airflow.utils.email.send_email_smtp function, you have to configure an
+# smtp server here
+
+# Specifies the host server address used by Airflow when sending out email
notifications via SMTP.
+#
+# Variable: AIRFLOW__SMTP__SMTP_HOST
+#
+smtp_host = localhost
+
+# Determines whether to use the STARTTLS command when connecting to the SMTP
server.
+#
+# Variable: AIRFLOW__SMTP__SMTP_STARTTLS
+#
+smtp_starttls = True
+
+# Determines whether to use an SSL connection when talking to the SMTP server.
+#
+# Variable: AIRFLOW__SMTP__SMTP_SSL
+#
+smtp_ssl = False
+
+# Username to authenticate when connecting to smtp server.
+#
+# Example: smtp_user = airflow
+#
+# Variable: AIRFLOW__SMTP__SMTP_USER
+#
+# smtp_user =
+
+# Password to authenticate when connecting to smtp server.
+#
+# Example: smtp_password = airflow
+#
+# Variable: AIRFLOW__SMTP__SMTP_PASSWORD
+#
+# smtp_password =
+
+# Defines the port number on which Airflow connects to the SMTP server to send
email notifications.
+#
+# Variable: AIRFLOW__SMTP__SMTP_PORT
+#
+smtp_port = 25
+
+# Specifies the default **from** email address used when Airflow sends email
notifications.
+#
+# Variable: AIRFLOW__SMTP__SMTP_MAIL_FROM
+#
+smtp_mail_from = airf...@example.com
+
+# Determines the maximum time (in seconds) the Apache Airflow system will wait
for a
+# connection to the SMTP server to be established.
+#
+# Variable: AIRFLOW__SMTP__SMTP_TIMEOUT
+#
+smtp_timeout = 30
+
+# Defines the maximum number of times Airflow will attempt to connect to the
SMTP server.
+#
+# Variable: AIRFLOW__SMTP__SMTP_RETRY_LIMIT
+#
+smtp_retry_limit = 5
+
+[sentry]
+# `Sentry <https://docs.sentry.io>`__ integration. Here you can supply
+# additional configuration options based on the Python platform.
+# See `Python / Configuration / Basic Options
+# <https://docs.sentry.io/platforms/python/configuration/options/>`__ for more
details.
+# Unsupported options: ``integrations``, ``in_app_include``,
``in_app_exclude``,
+# ``ignore_errors``, ``before_breadcrumb``, ``transport``.
+
+# Enable error reporting to Sentry
+#
+# Variable: AIRFLOW__SENTRY__SENTRY_ON
+#
+sentry_on = false
+
+#
+# Variable: AIRFLOW__SENTRY__SENTRY_DSN
+#
+sentry_dsn =
+
+# Dotted path to a before_send function that the sentry SDK should be
configured to use.
+#
+# Variable: AIRFLOW__SENTRY__BEFORE_SEND
+#
+# before_send =
+
+[scheduler]
+# Task instances listen for external kill signal (when you clear tasks
+# from the CLI or the UI), this defines the frequency at which they should
+# listen (in seconds).
+#
+# Variable: AIRFLOW__SCHEDULER__JOB_HEARTBEAT_SEC
+#
+job_heartbeat_sec = 5
+
+# The scheduler constantly tries to trigger new tasks (look at the
+# scheduler section in the docs for more information). This defines
+# how often the scheduler should run (in seconds).
+#
+# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC
+#
+scheduler_heartbeat_sec = 5
+
+# The frequency (in seconds) at which the LocalTaskJob should send heartbeat
signals to the
+# scheduler to notify it's still alive. If this value is set to 0, the
heartbeat interval will default
+# to the value of ``[scheduler] scheduler_zombie_task_threshold``.
+#
+# Variable: AIRFLOW__SCHEDULER__LOCAL_TASK_JOB_HEARTBEAT_SEC
+#
+local_task_job_heartbeat_sec = 0
+
+# The number of times to try to schedule each DAG file
+# -1 indicates unlimited number
+#
+# Variable: AIRFLOW__SCHEDULER__NUM_RUNS
+#
+num_runs = -1
+
+# Controls how long the scheduler will sleep between loops, but if there was
nothing to do
+# in the loop. i.e. if it scheduled something then it will start the next loop
+# iteration straight away.
+#
+# Variable: AIRFLOW__SCHEDULER__SCHEDULER_IDLE_SLEEP_TIME
+#
+scheduler_idle_sleep_time = 1
+
+# Number of seconds after which a DAG file is parsed. The DAG file is parsed
every
+# ``[scheduler] min_file_process_interval`` number of seconds. Updates to DAGs
are reflected after
+# this interval. Keeping this number low will increase CPU usage.
+#
+# Variable: AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL
+#
+min_file_process_interval = 30
+
+# How often (in seconds) to check for stale DAGs (DAGs which are no longer
present in
+# the expected files) which should be deactivated, as well as datasets that
are no longer
+# referenced and should be marked as orphaned.
+#
+# Variable: AIRFLOW__SCHEDULER__PARSING_CLEANUP_INTERVAL
+#
+parsing_cleanup_interval = 60
+
+# How long (in seconds) to wait after we have re-parsed a DAG file before
deactivating stale
+# DAGs (DAGs which are no longer present in the expected files). The reason
why we need
+# this threshold is to account for the time between when the file is parsed
and when the
+# DAG is loaded. The absolute maximum that this could take is ``[core]
dag_file_processor_timeout``,
+# but when you have a long timeout configured, it results in a significant
delay in the
+# deactivation of stale dags.
+#
+# Variable: AIRFLOW__SCHEDULER__STALE_DAG_THRESHOLD
+#
+stale_dag_threshold = 50
+
+# How often (in seconds) to scan the DAGs directory for new files. Default to
5 minutes.
+#
+# Variable: AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL
+#
+dag_dir_list_interval = 300
+
+# How often should stats be printed to the logs. Setting to 0 will disable
printing stats
+#
+# Variable: AIRFLOW__SCHEDULER__PRINT_STATS_INTERVAL
+#
+print_stats_interval = 30
+
+# How often (in seconds) should pool usage stats be sent to StatsD (if
statsd_on is enabled)
+#
+# Variable: AIRFLOW__SCHEDULER__POOL_METRICS_INTERVAL
+#
+pool_metrics_interval = 5.0
+
+# If the last scheduler heartbeat happened more than ``[scheduler]
scheduler_health_check_threshold``
+# ago (in seconds), scheduler is considered unhealthy.
+# This is used by the health check in the **/health** endpoint and in
``airflow jobs check`` CLI
+# for SchedulerJob.
+#
+# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_THRESHOLD
+#
+scheduler_health_check_threshold = 30
+
+# When you start a scheduler, airflow starts a tiny web server
+# subprocess to serve a health check if this is set to ``True``
+#
+# Variable: AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK
+#
+enable_health_check = False
+
+# When you start a scheduler, airflow starts a tiny web server
+# subprocess to serve a health check on this host
+#
+# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_SERVER_HOST
+#
+scheduler_health_check_server_host = 0.0.0.0
+
+# When you start a scheduler, airflow starts a tiny web server
+# subprocess to serve a health check on this port
+#
+# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_SERVER_PORT
+#
+scheduler_health_check_server_port = 8974
+
+# How often (in seconds) should the scheduler check for orphaned tasks and
SchedulerJobs
+#
+# Variable: AIRFLOW__SCHEDULER__ORPHANED_TASKS_CHECK_INTERVAL
+#
+orphaned_tasks_check_interval = 300.0
+
+# Determines the directory where logs for the child processes of the scheduler
will be stored
+#
+# Variable: AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY
+#
+child_process_log_directory = /var/lib/airflow/logs/scheduler
+
+# Local task jobs periodically heartbeat to the DB. If the job has
+# not heartbeat in this many seconds, the scheduler will mark the
+# associated task instance as failed and will re-schedule the task.
+#
+# Variable: AIRFLOW__SCHEDULER__SCHEDULER_ZOMBIE_TASK_THRESHOLD
+#
+scheduler_zombie_task_threshold = 300
+
+# How often (in seconds) should the scheduler check for zombie tasks.
+#
+# Variable: AIRFLOW__SCHEDULER__ZOMBIE_DETECTION_INTERVAL
+#
+zombie_detection_interval = 10.0
+
+# Turn off scheduler catchup by setting this to ``False``.
+# Default behavior is unchanged and
+# Command Line Backfills still work, but the scheduler
+# will not do scheduler catchup if this is ``False``,
+# however it can be set on a per DAG basis in the
+# DAG definition (catchup)
+#
+# Variable: AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT
+#
+catchup_by_default = True
+
+# Setting this to ``True`` will make first task instance of a task
+# ignore depends_on_past setting. A task instance will be considered
+# as the first task instance of a task when there is no task instance
+# in the DB with an execution_date earlier than it., i.e. no manual marking
+# success will be needed for a newly added task to be scheduled.
+#
+# Variable: AIRFLOW__SCHEDULER__IGNORE_FIRST_DEPENDS_ON_PAST_BY_DEFAULT
+#
+ignore_first_depends_on_past_by_default = True
+
+# This changes the batch size of queries in the scheduling main loop.
+# This should not be greater than ``[core] parallelism``.
+# If this is too high, SQL query performance may be impacted by
+# complexity of query predicate, and/or excessive locking.
+# Additionally, you may hit the maximum allowable query length for your db.
+# Set this to 0 to use the value of ``[core] parallelism``
+#
+# Variable: AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY
+#
+max_tis_per_query = 16
+
+# Should the scheduler issue ``SELECT ... FOR UPDATE`` in relevant queries.
+# If this is set to ``False`` then you should not run more than a single
+# scheduler at once
+#
+# Variable: AIRFLOW__SCHEDULER__USE_ROW_LEVEL_LOCKING
+#
+use_row_level_locking = True
+
+# Max number of DAGs to create DagRuns for per scheduler loop.
+#
+# Variable: AIRFLOW__SCHEDULER__MAX_DAGRUNS_TO_CREATE_PER_LOOP
+#
+max_dagruns_to_create_per_loop = 10
+
+# How many DagRuns should a scheduler examine (and lock) when scheduling
+# and queuing tasks.
+#
+# Variable: AIRFLOW__SCHEDULER__MAX_DAGRUNS_PER_LOOP_TO_SCHEDULE
+#
+max_dagruns_per_loop_to_schedule = 20
+
+# Should the Task supervisor process perform a "mini scheduler" to attempt to
schedule more tasks of the
+# same DAG. Leaving this on will mean tasks in the same DAG execute quicker,
but might starve out other
+# dags in some circumstances
+#
+# Variable: AIRFLOW__SCHEDULER__SCHEDULE_AFTER_TASK_EXECUTION
+#
+schedule_after_task_execution = True
+
+# The scheduler reads dag files to extract the airflow modules that are going
to be used,
+# and imports them ahead of time to avoid having to re-do it for each parsing
process.
+# This flag can be set to ``False`` to disable this behavior in case an
airflow module needs
+# to be freshly imported each time (at the cost of increased DAG parsing time).
+#
+# Variable: AIRFLOW__SCHEDULER__PARSING_PRE_IMPORT_MODULES
+#
+parsing_pre_import_modules = True
+
+# The scheduler can run multiple processes in parallel to parse dags.
+# This defines how many processes will run.
+#
+# Variable: AIRFLOW__SCHEDULER__PARSING_PROCESSES
+#
+parsing_processes = 2
+
+# One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``.
+# The scheduler will list and sort the dag files to decide the parsing order.
+#
+# * ``modified_time``: Sort by modified time of the files. This is useful on
large scale to parse the
+# recently modified DAGs first.
+# * ``random_seeded_by_host``: Sort randomly across multiple Schedulers but
with same order on the
+# same host. This is useful when running with Scheduler in HA mode where
each scheduler can
+# parse different DAG files.
+# * ``alphabetical``: Sort by filename
+#
+# Variable: AIRFLOW__SCHEDULER__FILE_PARSING_SORT_MODE
+#
+file_parsing_sort_mode = modified_time
+
+# Whether the dag processor is running as a standalone process or it is a
subprocess of a scheduler
+# job.
+#
+# Variable: AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR
+#
+standalone_dag_processor = False
+
+# Only applicable if ``[scheduler] standalone_dag_processor`` is true and
callbacks are stored
+# in database. Contains maximum number of callbacks that are fetched during a
single loop.
+#
+# Variable: AIRFLOW__SCHEDULER__MAX_CALLBACKS_PER_LOOP
+#
+max_callbacks_per_loop = 20
+
+# Only applicable if ``[scheduler] standalone_dag_processor`` is true.
+# Time in seconds after which dags, which were not updated by Dag Processor
are deactivated.
+#
+# Variable: AIRFLOW__SCHEDULER__DAG_STALE_NOT_SEEN_DURATION
+#
+dag_stale_not_seen_duration = 600
+
+# Turn off scheduler use of cron intervals by setting this to ``False``.
+# DAGs submitted manually in the web UI or with trigger_dag will still run.
+#
+# Variable: AIRFLOW__SCHEDULER__USE_JOB_SCHEDULE
+#
+use_job_schedule = True
+
+# Allow externally triggered DagRuns for Execution Dates in the future
+# Only has effect if schedule_interval is set to None in DAG
+#
+# Variable: AIRFLOW__SCHEDULER__ALLOW_TRIGGER_IN_FUTURE
+#
+allow_trigger_in_future = False
+
+# How often to check for expired trigger requests that have not run yet.
+#
+# Variable: AIRFLOW__SCHEDULER__TRIGGER_TIMEOUT_CHECK_INTERVAL
+#
+trigger_timeout_check_interval = 15
+
+# Amount of time a task can be in the queued state before being retried or set
to failed.
+#
+# Variable: AIRFLOW__SCHEDULER__TASK_QUEUED_TIMEOUT
+#
+task_queued_timeout = 600.0
+
+# How often to check for tasks that have been in the queued state for
+# longer than ``[scheduler] task_queued_timeout``.
+#
+# Variable: AIRFLOW__SCHEDULER__TASK_QUEUED_TIMEOUT_CHECK_INTERVAL
+#
+task_queued_timeout_check_interval = 120.0
+
+# The run_id pattern used to verify the validity of user input to the run_id
parameter when
+# triggering a DAG. This pattern cannot change the pattern used by scheduler
to generate run_id
+# for scheduled DAG runs or DAG runs triggered without changing the run_id
parameter.
+#
+# Variable: AIRFLOW__SCHEDULER__ALLOWED_RUN_ID_PATTERN
+#
+allowed_run_id_pattern = ^[A-Za-z0-9_.~:+-]+$
+
+# Whether to create DAG runs that span an interval or one single point in time
for cron schedules, when
+# a cron string is provided to ``schedule`` argument of a DAG.
+#
+# * ``True``: **CronDataIntervalTimetable** is used, which is suitable
+# for DAGs with well-defined data interval. You get contiguous intervals
from the end of the previous
+# interval up to the scheduled datetime.
+# * ``False``: **CronTriggerTimetable** is used, which is closer to the
behavior of cron itself.
+#
+# Notably, for **CronTriggerTimetable**, the logical date is the same as the
time the DAG Run will
+# try to schedule, while for **CronDataIntervalTimetable**, the logical date
is the beginning of
+# the data interval, but the DAG Run will try to schedule at the end of the
data interval.
+#
+# Variable: AIRFLOW__SCHEDULER__CREATE_CRON_DATA_INTERVALS
+#
+create_cron_data_intervals = True
+
+[triggerer]
+# How many triggers a single Triggerer will run at once, by default.
+#
+# Variable: AIRFLOW__TRIGGERER__DEFAULT_CAPACITY
+#
+default_capacity = 1000
+
+# How often to heartbeat the Triggerer job to ensure it hasn't been killed.
+#
+# Variable: AIRFLOW__TRIGGERER__JOB_HEARTBEAT_SEC
+#
+job_heartbeat_sec = 5
+
+# If the last triggerer heartbeat happened more than ``[triggerer]
triggerer_health_check_threshold``
+# ago (in seconds), triggerer is considered unhealthy.
+# This is used by the health check in the **/health** endpoint and in
``airflow jobs check`` CLI
+# for TriggererJob.
+#
+# Variable: AIRFLOW__TRIGGERER__TRIGGERER_HEALTH_CHECK_THRESHOLD
+#
+triggerer_health_check_threshold = 30
+
+[kerberos]
+# Location of your ccache file once kinit has been performed.
+#
+# Variable: AIRFLOW__KERBEROS__CCACHE
+#
+ccache = /tmp/airflow_krb5_ccache
+
+# gets augmented with fqdn
+#
+# Variable: AIRFLOW__KERBEROS__PRINCIPAL
+#
+principal = airflow
+
+# Determines the frequency at which initialization or re-initialization
processes occur.
+#
+# Variable: AIRFLOW__KERBEROS__REINIT_FREQUENCY
+#
+reinit_frequency = 3600
+
+# Path to the kinit executable
+#
+# Variable: AIRFLOW__KERBEROS__KINIT_PATH
+#
+kinit_path = kinit
+
+# Designates the path to the Kerberos keytab file for the Airflow user
+#
+# Variable: AIRFLOW__KERBEROS__KEYTAB
+#
+keytab = airflow.keytab
+
+# Allow to disable ticket forwardability.
+#
+# Variable: AIRFLOW__KERBEROS__FORWARDABLE
+#
+forwardable = True
+
+# Allow to remove source IP from token, useful when using token behind NATted
Docker host.
+#
+# Variable: AIRFLOW__KERBEROS__INCLUDE_IP
+#
+include_ip = True
+
+[sensors]
+# Sensor default timeout, 7 days by default (7 * 24 * 60 * 60).
+#
+# Variable: AIRFLOW__SENSORS__DEFAULT_TIMEOUT
+#
+default_timeout = 604800
+
+[usage_data_collection]
+# Airflow integrates `Scarf <https://about.scarf.sh/>`__ to collect basic
platform and usage data
+# during operation. This data assists Airflow maintainers in better
understanding how Airflow is used.
+# Insights gained from this telemetry are critical for prioritizing patches,
minor releases, and
+# security fixes. Additionally, this information supports key decisions
related to the development road map.
+# Check the FAQ doc for more information on what data is collected.
+#
+# Deployments can opt-out of analytics by setting the ``enabled`` option
+# to ``False``, or the ``SCARF_ANALYTICS=false`` environment variable.
+# Individual users can easily opt-out of analytics in various ways documented
in the
+# `Scarf Do Not Track docs <https://docs.scarf.sh/gateway/#do-not-track>`__.
+
+# Enable or disable usage data collection and sending.
+#
+# Variable: AIRFLOW__USAGE_DATA_COLLECTION__ENABLED
+#
+enabled = True
+
+[common.io]
+# Common IO configuration section
+
+# Path to a location on object storage where XComs can be stored in url format.
+#
+# Example: xcom_objectstorage_path = s3://conn_id@bucket/path
+#
+# Variable: AIRFLOW__COMMON.IO__XCOM_OBJECTSTORAGE_PATH
+#
+xcom_objectstorage_path =
+
+# Threshold in bytes for storing XComs in object storage. -1 means always
store in the
+# database. 0 means always store in object storage. Any positive number means
+# it will be stored in object storage if the size of the value is greater than
the threshold.
+#
+# Example: xcom_objectstorage_threshold = 1000000
+#
+# Variable: AIRFLOW__COMMON.IO__XCOM_OBJECTSTORAGE_THRESHOLD
+#
+xcom_objectstorage_threshold = -1
+
+# Compression algorithm to use when storing XComs in object storage. Supported
algorithms
+# are a.o.: snappy, zip, gzip, bz2, and lzma. If not specified, no compression
will be used.
+# Note that the compression algorithm must be available in the Python
installation (e.g.
+# python-snappy for snappy). Zip, gz, bz2 are available by default.
+#
+# Example: xcom_objectstorage_compression = gz
+#
+# Variable: AIRFLOW__COMMON.IO__XCOM_OBJECTSTORAGE_COMPRESSION
+#
+xcom_objectstorage_compression =
+
+[fab]
+# This section contains configs specific to FAB provider.
+
+# Boolean for enabling rate limiting on authentication endpoints.
+#
+# Variable: AIRFLOW__FAB__AUTH_RATE_LIMITED
+#
+auth_rate_limited = True
+
+# Rate limit for authentication endpoints.
+#
+# Variable: AIRFLOW__FAB__AUTH_RATE_LIMIT
+#
+auth_rate_limit = 5 per 40 second
+
+# Update FAB permissions and sync security manager roles
+# on webserver startup
+#
+# Variable: AIRFLOW__FAB__UPDATE_FAB_PERMS
+#
+update_fab_perms = True
+
+[imap]
+# Options for IMAP provider.
+
+# ssl_context =
+
+[smtp_provider]
+# Options for SMTP provider.
+
+# ssl context to use when using SMTP and IMAP SSL connections. By default, the
context is "default"
+# which sets it to ``ssl.create_default_context()`` which provides the right
balance between
+# compatibility and security, it however requires that certificates in your
operating system are
+# updated and that SMTP/IMAP servers of yours have valid certificates that
have corresponding public
+# keys installed on your machines. You can switch it to "none" if you want to
disable checking
+# of the certificates, but it is not recommended as it allows MITM
(man-in-the-middle) attacks
+# if your infrastructure is not sufficiently secured. It should only be set
temporarily while you
+# are fixing your certificate configuration. This can be typically done by
upgrading to newer
+# version of the operating system you run Airflow components on,by
upgrading/refreshing proper
+# certificates in the OS or by updating certificates for your mail servers.
+#
+# If you do not set this option explicitly, it will use Airflow
"email.ssl_context" configuration,
+# but if this configuration is not present, it will use "default" value.
+#
+# Example: ssl_context = default
+#
+# Variable: AIRFLOW__SMTP_PROVIDER__SSL_CONTEXT
+#
+# ssl_context =
+
+# Allows overriding of the standard templated email subject line when the
SmtpNotifier is used.
+# Must provide a path to the template.
+#
+# Example: templated_email_subject_path = path/to/override/email_subject.html
+#
+# Variable: AIRFLOW__SMTP_PROVIDER__TEMPLATED_EMAIL_SUBJECT_PATH
+#
+# templated_email_subject_path =
+
+# Allows overriding of the standard templated email path when the SmtpNotifier
is used. Must provide
+# a path to the template.
+#
+# Example: templated_html_content_path = path/to/override/email.html
+#
+# Variable: AIRFLOW__SMTP_PROVIDER__TEMPLATED_HTML_CONTENT_PATH
+#
+# templated_html_content_path =
+
diff --git a/bigtop-packages/src/common/airflow/airflow-scheduler.service
b/bigtop-packages/src/common/airflow/airflow-scheduler.service
new file mode 100644
index 000000000..a2106f1ed
--- /dev/null
+++ b/bigtop-packages/src/common/airflow/airflow-scheduler.service
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[Unit]
+Description=Airflow scheduler daemon
+After=network.target postgresql.service mysql.service redis.service
rabbitmq-server.service
+Wants=postgresql.service mysql.service redis.service rabbitmq-server.service
+
+[Service]
+EnvironmentFile=/etc/default/airflow
+User=airflow
+Group=airflow
+Type=simple
+ExecStart=/usr/lib/airflow/bin/airflow scheduler
+Restart=always
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target
diff --git a/bigtop-packages/src/common/airflow/airflow-webserver.service
b/bigtop-packages/src/common/airflow/airflow-webserver.service
new file mode 100644
index 000000000..6f194ef0f
--- /dev/null
+++ b/bigtop-packages/src/common/airflow/airflow-webserver.service
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[Unit]
+Description=Airflow webserver daemon
+After=network.target postgresql.service mysql.service redis.service
rabbitmq-server.service
+Wants=postgresql.service mysql.service redis.service rabbitmq-server.service
+
+[Service]
+EnvironmentFile=/etc/default/airflow
+User=airflow
+Group=airflow
+Type=simple
+ExecStart=/usr/lib/airflow/bin/airflow webserver --pid
/run/airflow/webserver.pid
+Restart=on-failure
+RestartSec=5s
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target
diff --git a/bigtop-packages/src/common/airflow/airflow.conf
b/bigtop-packages/src/common/airflow/airflow.conf
new file mode 100644
index 000000000..03dc9e2f3
--- /dev/null
+++ b/bigtop-packages/src/common/airflow/airflow.conf
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+d /run/airflow 0755 airflow airflow
diff --git a/bigtop-packages/src/common/airflow/airflow.default
b/bigtop-packages/src/common/airflow/airflow.default
new file mode 100644
index 000000000..3c73385b5
--- /dev/null
+++ b/bigtop-packages/src/common/airflow/airflow.default
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This file is the environment file for Airflow. Put this file in
/etc/sysconfig/airflow per default
+# configuration of the systemd unit files.
+#
+# AIRFLOW_CONFIG=
+AIRFLOW_HOME=/var/lib/airflow
+PATH=/usr/lib/airflow/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
diff --git a/bigtop-packages/src/common/airflow/do-component-build
b/bigtop-packages/src/common/airflow/do-component-build
new file mode 100644
index 000000000..dc8458809
--- /dev/null
+++ b/bigtop-packages/src/common/airflow/do-component-build
@@ -0,0 +1,24 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -ex
+
+python3 -m venv --copies --upgrade-deps build
+source build/bin/activate
+python3 -m pip install .
+deactivate
+
+find build/lib -name '*.pyc' -type f -delete
+find build/lib -name __pycache__ -type d -delete
diff --git a/bigtop-packages/src/common/airflow/install_airflow.sh
b/bigtop-packages/src/common/airflow/install_airflow.sh
new file mode 100644
index 000000000..413143a95
--- /dev/null
+++ b/bigtop-packages/src/common/airflow/install_airflow.sh
@@ -0,0 +1,77 @@
+#!/bin/bash -x
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+usage() {
+ echo "
+usage: $0 <options>
+ Required not-so-options:
+ --distro-dir=DIR path to distro specific files (debian/RPM)
+ --build-dir=DIR path to build directory
+ --prefix=PREFIX path to install into
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'prefix:' \
+ -l 'distro-dir:' \
+ -l 'build-dir:' \
+ -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "$OPTS"
+
+while true ; do
+ case "$1" in
+ --prefix)
+ PREFIX=$2 ; shift 2
+ ;;
+ --distro-dir)
+ DISTRO_DIR=$2 ; shift 2
+ ;;
+ --build-dir)
+ BUILD_DIR=$2 ; shift 2
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+install -d -p -m 755 ${PREFIX}/etc/default
+install -d -p -m 755 ${PREFIX}/usr/lib/airflow
+install -d -p -m 755 ${PREFIX}/usr/lib/systemd/system
+install -d -p -m 755 ${PREFIX}/usr/lib/tmpfiles.d
+
+find ${BUILD_DIR}/bin -type f -exec sed -i -e "s,$(cd ${BUILD_DIR} &&
pwd),/usr/lib/airflow," {} \;
+find ${BUILD_DIR}/lib -type f -exec sed -i -e "s,/usr/bin/env
python$,/usr/bin/env python3," {} \;
+
+cp -r ${BUILD_DIR}/{bin,include,lib,pyvenv.cfg} ${PREFIX}/usr/lib/airflow
+cp ${DISTRO_DIR}/airflow.default ${PREFIX}/etc/default/airflow
+cp ${DISTRO_DIR}/airflow-{scheduler,webserver}.service
${PREFIX}/usr/lib/systemd/system
+cp ${DISTRO_DIR}/airflow.conf ${PREFIX}/usr/lib/tmpfiles.d
diff --git a/bigtop-packages/src/deb/airflow/airflow.install
b/bigtop-packages/src/deb/airflow/airflow.install
new file mode 100644
index 000000000..c9ff3bde4
--- /dev/null
+++ b/bigtop-packages/src/deb/airflow/airflow.install
@@ -0,0 +1,5 @@
+/etc/default/airflow
+/usr/lib/airflow
+/usr/lib/systemd/system/airflow-scheduler.service
+/usr/lib/systemd/system/airflow-webserver.service
+/usr/lib/tmpfiles.d/airflow.conf
diff --git a/bigtop-packages/src/deb/airflow/airflow.postinst
b/bigtop-packages/src/deb/airflow/airflow.postinst
new file mode 100644
index 000000000..486ad6206
--- /dev/null
+++ b/bigtop-packages/src/deb/airflow/airflow.postinst
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+case "$1" in
+ configure)
+ chown -R airflow:airflow /var/lib/airflow
+ ;;
+
+ abort-upgrade|abort-remove|abort-deconfigure)
+ ;;
+
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
diff --git a/bigtop-packages/src/deb/airflow/airflow.preinst
b/bigtop-packages/src/deb/airflow/airflow.preinst
new file mode 100644
index 000000000..6dfec52f4
--- /dev/null
+++ b/bigtop-packages/src/deb/airflow/airflow.preinst
@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# summary of how this script can be called:
+# * <new-preinst> `install'
+# * <new-preinst> `install' <old-version>
+# * <new-preinst> `upgrade' <old-version>
+# * <old-preinst> `abort-upgrade' <new-version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+case "$1" in
+ install|upgrade)
+ getent group airflow >/dev/null || groupadd -r airflow
+ if ! getent passwd airflow >/dev/null; then
+ # Adding system user: airflow .
+ adduser \
+ --system \
+ --ingroup airflow \
+ --home /var/lib/airflow \
+ --gecos "Airflow User" \
+ --shell /bin/false \
+ airflow >/dev/null
+ fi
+ install -d -m 0755 -o airflow -g airflow /var/lib/airflow
+ ;;
+
+ abort-upgrade)
+ ;;
+
+ *)
+ echo "preinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
diff --git a/bigtop-packages/src/deb/airflow/changelog
b/bigtop-packages/src/deb/airflow/changelog
new file mode 100644
index 000000000..d4858cd49
--- /dev/null
+++ b/bigtop-packages/src/deb/airflow/changelog
@@ -0,0 +1 @@
+--- This is auto-generated
diff --git a/bigtop-packages/src/deb/airflow/compat
b/bigtop-packages/src/deb/airflow/compat
new file mode 100644
index 000000000..ec635144f
--- /dev/null
+++ b/bigtop-packages/src/deb/airflow/compat
@@ -0,0 +1 @@
+9
diff --git a/bigtop-packages/src/deb/airflow/control
b/bigtop-packages/src/deb/airflow/control
new file mode 100644
index 000000000..af75c059c
--- /dev/null
+++ b/bigtop-packages/src/deb/airflow/control
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Source: airflow
+Section: unknown
+Priority: optional
+Maintainer: Bigtop <d...@bigtop.apache.org>
+Build-Depends: debhelper (>=9)
+Standards-Version: 3.9.6
+Homepage: https://airflow.apache.org/
+
+Package: airflow
+Architecture: any
+Depends: libexpat1, libpython3-stdlib
+Description: Apache Airflow is a platform to programmatically
+ author, schedule, and monitor workflows.
diff --git a/bigtop-packages/src/deb/airflow/copyright
b/bigtop-packages/src/deb/airflow/copyright
new file mode 100644
index 000000000..a796fa6f3
--- /dev/null
+++ b/bigtop-packages/src/deb/airflow/copyright
@@ -0,0 +1,11 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: airflow
+Source: https://airflow.apache.org/
+
+Files: *
+Copyright: 2016-2025, The Apache Software Foundation
+License: Apache-2.0
+
+License: Apache-2.0
+ On Debian systems, the complete text of the Apache 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".
diff --git a/bigtop-packages/src/deb/airflow/rules
b/bigtop-packages/src/deb/airflow/rules
new file mode 100755
index 000000000..684ccb315
--- /dev/null
+++ b/bigtop-packages/src/deb/airflow/rules
@@ -0,0 +1,37 @@
+#!/usr/bin/make -f
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Uncomment this to turn on verbose mode.
+export DH_VERBOSE=1
+
+# This has to be exported to make some magic below work.
+export DH_OPTIONS
+
+%:
+ dh $@
+
+override_dh_auto_build:
+ bash debian/do-component-build
+
+override_dh_auto_install:
+ bash debian/install_airflow.sh \
+ --prefix=debian/tmp/ \
+ --distro-dir=debian \
+ --build-dir=build
+
+override_dh_strip:
+ dh_strip --no-automatic-dbgsym
\ No newline at end of file
diff --git a/bigtop-packages/src/deb/airflow/source/format
b/bigtop-packages/src/deb/airflow/source/format
new file mode 100644
index 000000000..163aaf8d8
--- /dev/null
+++ b/bigtop-packages/src/deb/airflow/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
diff --git a/bigtop-packages/src/rpm/airflow/SPECS/airflow.spec
b/bigtop-packages/src/rpm/airflow/SPECS/airflow.spec
new file mode 100644
index 000000000..3331d9f53
--- /dev/null
+++ b/bigtop-packages/src/rpm/airflow/SPECS/airflow.spec
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+%global debug_package %{nil}
+%define _build_id_links none
+
+%define airflow_name airflow
+%define airflow_pkg_name %{airflow_name}%{pkg_name_suffix}
+%define lib_dir /usr/lib/airflow
+%define var_dir /var/lib/airflow
+
+Name: %{airflow_name}
+Version: %{airflow_version}
+Release: %{airflow_release}
+Summary: Apache Airflow
+Group: Applications/Engineering
+License: ASL 2.0
+URL: https://airflow.apache.org/
+BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
+Source0: apache_%{airflow_name}-%{airflow_base_version}.tar.gz
+Source1: do-component-build
+Source2: install_airflow.sh
+#BIGTOP_PATCH_FILES
+Requires: bash
+# "which" command is needed for a lot of projects.
+# It is part of the package "util-linux" on suse and "which" everywhere else
+%if %{?suse_version:1}0
+Requires: util-linux
+%else
+Requires: which
+%endif
+
+%description
+Apache Airflow is a platform to programmatically author, schedule, and monitor
workflows.
+
+%prep
+%setup -n apache_%{airflow_name}-%{airflow_base_version}
+
+#BIGTOP_PATCH_COMMANDS
+
+%build
+bash %{SOURCE1}
+
+%install
+%__rm -rf $RPM_BUILD_ROOT
+bash %{SOURCE2} \
+ --distro-dir=${RPM_SOURCE_DIR} \
+ --build-dir=build \
+ --prefix=${RPM_BUILD_ROOT}
+
+%pre
+getent group airflow >/dev/null || groupadd -r airflow
+getent passwd airflow > /dev/null || useradd -c "airflow" -s /sbin/nologin -g
airflow -r -d %{var_dir} airflow 2> /dev/null || :
+
+%post
+AIRFLOW_HOME=%{var_dir} %{lib_dir}/bin/airflow db init
+chown -R airflow:airflow %{var_dir}
+
+%files
+%defattr(-,root,root)
+%{lib_dir}
+
+%attr(0644,root,root) %{_unitdir}/airflow-scheduler.service
+%attr(0644,root,root) %{_unitdir}/airflow-webserver.service
+
+%config(noreplace) /etc/default/airflow
+%config(noreplace) %{_tmpfilesdir}/airflow.conf
diff --git a/bigtop-tests/smoke-tests/airflow/TestAirflow.groovy
b/bigtop-tests/smoke-tests/airflow/TestAirflow.groovy
new file mode 100644
index 000000000..9f89194a7
--- /dev/null
+++ b/bigtop-tests/smoke-tests/airflow/TestAirflow.groovy
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.airflow
+
+import groovy.json.JsonSlurper
+import org.junit.Test
+import static org.junit.Assert.assertEquals
+
+class TestAirflow {
+ @Test
+ void testSchedulerAndWebserver() {
+ def json = new URL('http://localhost:8080/health').getText()
+ def health = new JsonSlurper().parseText(json)
+ assertEquals(health.get("scheduler").get("status"), "healthy")
+ }
+}
diff --git a/bigtop-tests/smoke-tests/airflow/build.gradle
b/bigtop-tests/smoke-tests/airflow/build.gradle
new file mode 100644
index 000000000..f1f5b9cbc
--- /dev/null
+++ b/bigtop-tests/smoke-tests/airflow/build.gradle
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+def tests_to_include() {
+ return [
+ "TestAirflow.groovy"
+ ];
+}
+
+sourceSets {
+ test {
+ groovy {
+ srcDirs = ["${BIGTOP_HOME}/bigtop-tests/smoke-tests/airflow/"]
+ }
+ }
+}
+
+test.doFirst {
+ checkEnv(["AIRFLOW_HOME"])
+}
diff --git a/bigtop.bom b/bigtop.bom
index ef5e93278..245ab0727 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -346,5 +346,15 @@ bigtop {
archive = site }
maven_parallel_build = true
}
+ 'airflow' {
+ name = "airflow"
+ relNotes = "Apache Airflow"
+ version { base = '2.10.4'; pkg = base; release = 1 }
+ tarball { source = "apache_airflow-${version.base}.tar.gz"
+ destination = source }
+ url { download_path = "/${name}/${version.base}/"
+ site = "${apache.APACHE_MIRROR}/${download_path}"
+ archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
+ }
}
}
diff --git a/bigtop_toolchain/manifests/python.pp
b/bigtop_toolchain/manifests/python.pp
index f1d9aaa34..33b5dfdab 100644
--- a/bigtop_toolchain/manifests/python.pp
+++ b/bigtop_toolchain/manifests/python.pp
@@ -27,6 +27,9 @@ class bigtop_toolchain::python {
package { 'python3-dev' :
ensure => present
}
+ package { 'python3-full' :
+ ensure => present
+ }
}
}
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
index fd7ce07b1..8a04ad324 100755
--- a/provisioner/utils/smoke-tests.sh
+++ b/provisioner/utils/smoke-tests.sh
@@ -32,6 +32,7 @@ fi
echo -e "\n===== EXPORTING VARIABLES =====\n"
+export AIRFLOW_HOME=${AIRFLOW_HOME:-/var/lib/airflow}
export ALLUXIO_HOME=${ALLUXIO_HOME:-/usr/lib/alluxio}
export FLINK_HOME=${FLINK_HOME:-/usr/lib/flink}
export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}
