This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b672ba478c Fix edge cases of "migrate/create-default-connections"
(#33136)
b672ba478c is described below
commit b672ba478cdf579d159d5ddd4c823a36e606f168
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Aug 6 00:19:45 2023 +0200
Fix edge cases of "migrate/create-default-connections" (#33136)
In #32810, "airflow db migrate" command has been added and it
is used by `start-airflow` command. There were a few edge cases
not covered and this PR completes it.
* We can move the "database/load_default_connections" configuration
to a new "deprecated" section. This is the first time we remove
the option completely as it lost its meaning, but we likely
still want to explain that the option was there and what it does
when deprecated "db init" command is used.
It has no meaning when you use "airflow db migrate" or when you run the
new "airflow connections create-default-connections" commands. So we can
now remove it completely from configuraiton. It will still work
in the "airflow db init" which is deprecated, as long as we provide an
explicit fallback. Also if someone had it defined in their config or
env variable, it will continue to work even if it is not defined.
* We need to explain the change in a significant newsfragment.
* The ``start-airflow`` command supports creating default connections
with ``--load-default-connections`` flag. This was lost after
the change so this PR brings it back by running the new
"airflow connections create-default-connections" command if the
flag is used.
* The `start-airflow` breeze command can be used to start older
versions of airflow - with ``--use-airflow-version" - those that do not
support `airflow db migrate` command. In this case the old behaviour is
used with setting the "AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS"
based on the flag passed and running "airflow db init" instead.
---
Dockerfile.ci | 1 -
airflow/cli/cli_config.py | 2 +-
airflow/cli/commands/db_command.py | 2 +-
airflow/config_templates/config.yml | 13 ++++++---
airflow/config_templates/config.yml.schema.json | 29 ++++++++++++++++----
airflow/config_templates/unit_tests.cfg | 3 +--
airflow/configuration.py | 9 ++++++-
airflow/utils/db.py | 1 -
dev/perf/sql_queries.py | 1 -
.../connections/aws.rst | 2 +-
docs/exts/includes/sections-and-options.rst | 31 ++++++++++++----------
docs/helm-chart/airflow-configuration.rst | 9 -------
newsfragments/33136.significant.rst | 5 ++++
scripts/docker/entrypoint_ci.sh | 1 -
scripts/in_container/check_environment.sh | 15 +++++++++--
tests/core/test_configuration.py | 1 -
16 files changed, 81 insertions(+), 44 deletions(-)
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 6c3e4b79de..10d786a2fa 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -929,7 +929,6 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
cd "${AIRFLOW_SOURCES}"
if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True"
]]; then
- export
AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
wait_for_asset_compilation
# shellcheck source=scripts/in_container/bin/run_tmux
diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py
index 4058eb4f86..52ed700db7 100644
--- a/airflow/cli/cli_config.py
+++ b/airflow/cli/cli_config.py
@@ -1520,7 +1520,7 @@ DB_COMMANDS = (
name="init",
help=(
"Deprecated -- use `migrate` instead. "
- "To create default connections use `connections
create-default-connections`. "
+ "To create default connections use `airflow connections
create-default-connections`. "
"Initialize the metadata database"
),
func=lazy_load_command("airflow.cli.commands.db_command.initdb"),
diff --git a/airflow/cli/commands/db_command.py
b/airflow/cli/commands/db_command.py
index 390d940fcf..2fb21dfd22 100644
--- a/airflow/cli/commands/db_command.py
+++ b/airflow/cli/commands/db_command.py
@@ -42,7 +42,7 @@ def initdb(args):
"""Initializes the metadata database."""
warnings.warn(
"`db init` is deprecated. Use `db migrate` instead to migrate the db
and/or "
- "create-default-connections to create the default connections",
+ "airflow connections create-default-connections to create the default
connections",
DeprecationWarning,
)
print("DB: " + repr(settings.engine.url))
diff --git a/airflow/config_templates/config.yml
b/airflow/config_templates/config.yml
index e5bce00987..bad34da8d5 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -591,10 +591,17 @@ database:
default: ~
load_default_connections:
description: |
- Whether to load the default connections that ship with Airflow. It's
good to
- get started, but you probably want to set this to ``False`` in a
production
- environment
+ Whether to load the default connections that ship with Airflow when
``airflow db init`` is called.
+ It's good to get started, but you probably want to set this to
``False`` in a production environment.
version_added: 2.3.0
+ version_deprecated: 2.7.0
+ deprecation_reason: |
+ This option is only used by the deprecated "airflow db init" command.
+ This option has been used in previous versions of Airflow to determine
if loading of the default
+ connections is done with the ``airflow db init`` command. This command
has been deprecated and
+ replaced by two separate commands ``airflow db migrate`` and
+ ``airflow connections create-default-connections`` and
``load_default_connections`` is not
+ used anymore by those commands.
type: string
example: ~
default: "True"
diff --git a/airflow/config_templates/config.yml.schema.json
b/airflow/config_templates/config.yml.schema.json
index 9bc321acc1..b89b2c00e7 100644
--- a/airflow/config_templates/config.yml.schema.json
+++ b/airflow/config_templates/config.yml.schema.json
@@ -38,13 +38,29 @@
"type": [
"string",
"null"
- ]
+ ],
+ "description": "Description of the option."
},
"version_added": {
"type": [
"string",
"null"
- ]
+ ],
+ "description": "When set to a version string, signals that
this option has been added in the version specified."
+ },
+ "version_deprecated": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "When set to a version string, this option
is deprecated as of this version, and will be removed in the future."
+ },
+ "deprecation_reason": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "The reason why this option is deprecated."
},
"type": {
"type": "string",
@@ -53,21 +69,24 @@
"boolean",
"integer",
"float"
- ]
+ ],
+ "description": "Type of the option - string, boolean,
integer or float."
},
"example": {
"type": [
"string",
"null",
"number"
- ]
+ ],
+ "description": "Example value for the option."
},
"default": {
"type": [
"string",
"null",
"number"
- ]
+ ],
+ "description": "Default value for the option."
},
"sensitive": {
"type": "boolean",
diff --git a/airflow/config_templates/unit_tests.cfg
b/airflow/config_templates/unit_tests.cfg
index 540c3f3be4..29d422704e 100644
--- a/airflow/config_templates/unit_tests.cfg
+++ b/airflow/config_templates/unit_tests.cfg
@@ -59,9 +59,8 @@ unit_test_mode = True
killed_task_cleanup_time = 5
# We only allow our own classes to be deserialized in tests
allowed_deserialization_classes = airflow\..* tests\..*
+
[database]
-# we want to have default connections loaded in unit tests
-load_default_connections = True
[logging]
# celery tests rely on it being set
diff --git a/airflow/configuration.py b/airflow/configuration.py
index a732f94f2d..57e8e5087d 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -960,6 +960,14 @@ class AirflowConfigParser(ConfigParser):
deprecated_section: str | None
deprecated_key: str | None
+ option_description = self.configuration_description.get(section,
{}).get(key, {})
+ if option_description.get("deprecated"):
+ deprecation_reason = option_description.get("deprecation_reason",
"")
+ warnings.warn(
+ f"The '{key}' option in section {section} is deprecated.
{deprecation_reason}",
+ DeprecationWarning,
+ stacklevel=2 + _extra_stacklevel,
+ )
# For when we rename whole sections
if section in self.inversed_deprecated_sections:
deprecated_section, deprecated_key = (section, key)
@@ -995,7 +1003,6 @@ class AirflowConfigParser(ConfigParser):
deprecated_section, deprecated_key, _ =
self.deprecated_options.get(
(section, key), (None, None, None)
)
-
# first check environment variables
option = self._get_environment_variables(
deprecated_key,
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index f0f60d1dc0..1a4a5db6ae 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -747,7 +747,6 @@ def initdb(session: Session = NEW_SESSION,
load_connections: bool = True):
upgradedb(session=session)
else:
_create_db_from_orm(session=session)
- # Load default connections
if conf.getboolean("database", "LOAD_DEFAULT_CONNECTIONS") and
load_connections:
create_default_connections(session=session)
# Add default pool & sync log_template
diff --git a/dev/perf/sql_queries.py b/dev/perf/sql_queries.py
index 2bddc349ef..76b87420eb 100644
--- a/dev/perf/sql_queries.py
+++ b/dev/perf/sql_queries.py
@@ -30,7 +30,6 @@ DAG_FOLDER = os.path.join(os.path.dirname(__file__), "dags")
os.environ["AIRFLOW__CORE__DAGS_FOLDER"] = DAG_FOLDER
os.environ["AIRFLOW__DEBUG__SQLALCHEMY_STATS"] = "True"
os.environ["AIRFLOW__CORE__LOAD_EXAMPLES"] = "False"
-os.environ["AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS"] = "True"
# Here we setup simpler logger to avoid any code changes in
# Airflow core code base
diff --git a/docs/apache-airflow-providers-amazon/connections/aws.rst
b/docs/apache-airflow-providers-amazon/connections/aws.rst
index 03c815f072..ba5fe01723 100644
--- a/docs/apache-airflow-providers-amazon/connections/aws.rst
+++ b/docs/apache-airflow-providers-amazon/connections/aws.rst
@@ -57,7 +57,7 @@ automatically the credentials from there.
This is no longer the case and the region needs to be set manually, either
in the connection screens in Airflow,
or via the ``AWS_DEFAULT_REGION`` environment variable.
-.. caution:: If you do not set ``[database] load_default_connections`` to
``True``
+.. caution:: If you do not run "airflow connections
create-default-connections" command,
most probably you do not have ``aws_default``. For historical reasons, the
Amazon Provider
components (Hooks, Operators, Sensors, etc.) fallback to the default boto3
credentials strategy
in case of a missing Connection ID. This behaviour is deprecated and will
be removed in a future releases.
diff --git a/docs/exts/includes/sections-and-options.rst
b/docs/exts/includes/sections-and-options.rst
index 497223fcee..33ea64231f 100644
--- a/docs/exts/includes/sections-and-options.rst
+++ b/docs/exts/includes/sections-and-options.rst
@@ -37,41 +37,44 @@
{% endif %}
{% for option_name, option in section["options"].items() %}
-
.. _config:{{ section_name }}__{{ option_name }}:
{{ option_name }}
{{ "-" * option_name|length }}
- {% if option["version_added"] %}
- .. versionadded:: {{ option["version_added"] }}
- {% endif %}
+ {% if option["version_added"] %}
+ .. versionadded:: {{ option["version_added"] }}
+ {% endif %}
- {% if option["description"] %}
+ {% if option["description"] %}
{{ option["description"] }}
- {% endif %}
+ {% endif %}
+
+ {% if option.get("version_deprecated") %}
+ .. deprecated:: {{ option["version_deprecated"] }}
+ {{ option["deprecation_reason"] | indent(width=8) }}
+ {% endif %}
- {% if option.get("see_also") %}
+ {% if option.get("see_also") %}
.. seealso:: {{ option["see_also"] }}
- {% endif %}
+ {% endif %}
:Type: {{ option["type"] }}
:Default: ``{{ "''" if option["default"] == "" else option["default"] }}``
- {% if option.get("sensitive") %}
+ {% if option.get("sensitive") %}
:Environment Variables:
``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}``
``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_CMD``
``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_SECRET``
- {% else %}
+ {% else %}
:Environment Variable: ``AIRFLOW__{{ section_name | upper }}__{{
option_name | upper }}``
- {% endif %}
- {% if option["example"] %}
+ {% endif %}
+ {% if option["example"] %}
:Example:
``{{ option["example"] }}``
- {% endif %}
-
+ {% endif %}
{% endfor %}
{% if section_name in deprecated_options %}
diff --git a/docs/helm-chart/airflow-configuration.rst
b/docs/helm-chart/airflow-configuration.rst
index 90ccd7d370..97aae726c1 100644
--- a/docs/helm-chart/airflow-configuration.rst
+++ b/docs/helm-chart/airflow-configuration.rst
@@ -39,12 +39,3 @@ configuration prior to installing and deploying the service.
The recommended way to load example DAGs using the official Docker image and
chart is to configure the ``AIRFLOW__CORE__LOAD_EXAMPLES`` environment variable
in ``extraEnv`` (see :doc:`Parameters reference <parameters-ref>`). The
official Docker image has ``AIRFLOW__CORE__LOAD_EXAMPLES=False``
set within the image, so you need to override it with an environment
variable when deploying the chart in order for the examples to be present.
-
-.. note::
-
- The ``AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS`` variable is not used by
the Chart. Airflow Helm Chart is
- intended to be used as production deployment and loading default connections
is not supposed to be handled
- during Chart installation. The Chart is intended to install and configure
the Apache Airflow software
- and create database structure, but not to fill-in the data which should be
managed by the users.
- The default connections are only meaningful when you want to have a ``quick
start`` with Airflow or
- do some development and adding the data via Helm Chart installation is not a
good idea.
diff --git a/newsfragments/33136.significant.rst
b/newsfragments/33136.significant.rst
new file mode 100644
index 0000000000..9f7cb3aa92
--- /dev/null
+++ b/newsfragments/33136.significant.rst
@@ -0,0 +1,5 @@
+The "airflow db init", "airflow db upgrade" commands and "[database]
load_default_connections" configuration options are deprecated.
+
+Instead, you should use "airflow db migrate" command to create or upgrade
database. This command will not create default connections.
+In order to create default connections you need to run "airflow connections
create-default-connections" explicitly,
+after running "airflow db migrate".
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index e4cf2b989c..73b8a20deb 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -316,7 +316,6 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
cd "${AIRFLOW_SOURCES}"
if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True"
]]; then
- export
AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
wait_for_asset_compilation
# shellcheck source=scripts/in_container/bin/run_tmux
diff --git a/scripts/in_container/check_environment.sh
b/scripts/in_container/check_environment.sh
index b23dc18dd2..67d4f1814e 100755
--- a/scripts/in_container/check_environment.sh
+++ b/scripts/in_container/check_environment.sh
@@ -110,12 +110,23 @@ function startairflow_if_requested() {
echo
echo "Starting Airflow"
echo
- export
AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
. "$( dirname "${BASH_SOURCE[0]}" )/configure_environment.sh"
- airflow db migrate
+ if airflow db migrate
+ then
+ if [[ ${LOAD_DEFAULT_CONNECTIONS=} == "true" ||
${LOAD_DEFAULT_CONNECTIONS=} == "True" ]]; then
+ echo
+ echo "${COLOR_BLUE}Creating default connections${COLOR_RESET}"
+ echo
+ airflow connections create-default-connections
+ fi
+ else
+ # For Airflow versions that do not support db migrate, we should
run airflow db init and
+ # set the removed AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS
+
AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} airflow
db init
+ fi
airflow users create -u admin -p admin -f Thor -l Adminstra -r Admin
-e [email protected]
. "$( dirname "${BASH_SOURCE[0]}" )/run_init_script.sh"
diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py
index a6229dc07c..3a24d33111 100644
--- a/tests/core/test_configuration.py
+++ b/tests/core/test_configuration.py
@@ -157,7 +157,6 @@ class TestConf:
# test display_source
cfg_dict = conf.as_dict(display_source=True)
assert cfg_dict["core"]["load_examples"][1] == "airflow.cfg"
- assert cfg_dict["database"]["load_default_connections"][1] ==
"airflow.cfg"
assert cfg_dict["testsection"]["testkey"] == ("testvalue", "env var")
assert cfg_dict["core"]["fernet_key"] == ("< hidden >", "env var")
