This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-7-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit b54f5030371ba4fcdc8437f06fc7916362e2a1a8 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Aug 6 00:19:45 2023 +0200 Fix edge cases of "migrate/create-default-connections" (#33136) In #32810, "airflow db migrate" command has been added and it is used by `start-airflow` command. There were a few edge cases not covered and this PR completes it. * We can move the "database/load_default_connections" configuration to a new "deprecated" section. This is the first time we remove the option completely as it lost its meaning, but we likely still want to explain that the option was there and what it does when deprecated "db init" command is used. It has no meaning when you use "airflow db migrate" or when you run the new "airflow connections create-default-connections" commands. So we can now remove it completely from configuraiton. It will still work in the "airflow db init" which is deprecated, as long as we provide an explicit fallback. Also if someone had it defined in their config or env variable, it will continue to work even if it is not defined. * We need to explain the change in a significant newsfragment. * The ``start-airflow`` command supports creating default connections with ``--load-default-connections`` flag. This was lost after the change so this PR brings it back by running the new "airflow connections create-default-connections" command if the flag is used. * The `start-airflow` breeze command can be used to start older versions of airflow - with ``--use-airflow-version" - those that do not support `airflow db migrate` command. In this case the old behaviour is used with setting the "AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS" based on the flag passed and running "airflow db init" instead. (cherry picked from commit b672ba478cdf579d159d5ddd4c823a36e606f168) --- Dockerfile.ci | 1 - airflow/cli/cli_config.py | 2 +- airflow/cli/commands/db_command.py | 2 +- airflow/config_templates/config.yml | 13 ++++++--- airflow/config_templates/config.yml.schema.json | 29 ++++++++++++++++---- airflow/config_templates/unit_tests.cfg | 3 +-- airflow/configuration.py | 9 ++++++- airflow/utils/db.py | 1 - dev/perf/sql_queries.py | 1 - .../connections/aws.rst | 2 +- docs/exts/includes/sections-and-options.rst | 31 ++++++++++++---------- docs/helm-chart/airflow-configuration.rst | 9 ------- newsfragments/33136.significant.rst | 5 ++++ scripts/docker/entrypoint_ci.sh | 1 - scripts/in_container/check_environment.sh | 15 +++++++++-- tests/core/test_configuration.py | 1 - 16 files changed, 81 insertions(+), 44 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 6c3e4b79de..10d786a2fa 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -929,7 +929,6 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then cd "${AIRFLOW_SOURCES}" if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then - export AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} wait_for_asset_compilation # shellcheck source=scripts/in_container/bin/run_tmux diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 4058eb4f86..52ed700db7 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -1520,7 +1520,7 @@ DB_COMMANDS = ( name="init", help=( "Deprecated -- use `migrate` instead. " - "To create default connections use `connections create-default-connections`. " + "To create default connections use `airflow connections create-default-connections`. " "Initialize the metadata database" ), func=lazy_load_command("airflow.cli.commands.db_command.initdb"), diff --git a/airflow/cli/commands/db_command.py b/airflow/cli/commands/db_command.py index 390d940fcf..2fb21dfd22 100644 --- a/airflow/cli/commands/db_command.py +++ b/airflow/cli/commands/db_command.py @@ -42,7 +42,7 @@ def initdb(args): """Initializes the metadata database.""" warnings.warn( "`db init` is deprecated. Use `db migrate` instead to migrate the db and/or " - "create-default-connections to create the default connections", + "airflow connections create-default-connections to create the default connections", DeprecationWarning, ) print("DB: " + repr(settings.engine.url)) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 3ec37a42e1..f61f6baca0 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -591,10 +591,17 @@ database: default: ~ load_default_connections: description: | - Whether to load the default connections that ship with Airflow. It's good to - get started, but you probably want to set this to ``False`` in a production - environment + Whether to load the default connections that ship with Airflow when ``airflow db init`` is called. + It's good to get started, but you probably want to set this to ``False`` in a production environment. version_added: 2.3.0 + version_deprecated: 2.7.0 + deprecation_reason: | + This option is only used by the deprecated "airflow db init" command. + This option has been used in previous versions of Airflow to determine if loading of the default + connections is done with the ``airflow db init`` command. This command has been deprecated and + replaced by two separate commands ``airflow db migrate`` and + ``airflow connections create-default-connections`` and ``load_default_connections`` is not + used anymore by those commands. type: string example: ~ default: "True" diff --git a/airflow/config_templates/config.yml.schema.json b/airflow/config_templates/config.yml.schema.json index 9bc321acc1..b89b2c00e7 100644 --- a/airflow/config_templates/config.yml.schema.json +++ b/airflow/config_templates/config.yml.schema.json @@ -38,13 +38,29 @@ "type": [ "string", "null" - ] + ], + "description": "Description of the option." }, "version_added": { "type": [ "string", "null" - ] + ], + "description": "When set to a version string, signals that this option has been added in the version specified." + }, + "version_deprecated": { + "type": [ + "string", + "null" + ], + "description": "When set to a version string, this option is deprecated as of this version, and will be removed in the future." + }, + "deprecation_reason": { + "type": [ + "string", + "null" + ], + "description": "The reason why this option is deprecated." }, "type": { "type": "string", @@ -53,21 +69,24 @@ "boolean", "integer", "float" - ] + ], + "description": "Type of the option - string, boolean, integer or float." }, "example": { "type": [ "string", "null", "number" - ] + ], + "description": "Example value for the option." }, "default": { "type": [ "string", "null", "number" - ] + ], + "description": "Default value for the option." }, "sensitive": { "type": "boolean", diff --git a/airflow/config_templates/unit_tests.cfg b/airflow/config_templates/unit_tests.cfg index 540c3f3be4..29d422704e 100644 --- a/airflow/config_templates/unit_tests.cfg +++ b/airflow/config_templates/unit_tests.cfg @@ -59,9 +59,8 @@ unit_test_mode = True killed_task_cleanup_time = 5 # We only allow our own classes to be deserialized in tests allowed_deserialization_classes = airflow\..* tests\..* + [database] -# we want to have default connections loaded in unit tests -load_default_connections = True [logging] # celery tests rely on it being set diff --git a/airflow/configuration.py b/airflow/configuration.py index a732f94f2d..57e8e5087d 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -960,6 +960,14 @@ class AirflowConfigParser(ConfigParser): deprecated_section: str | None deprecated_key: str | None + option_description = self.configuration_description.get(section, {}).get(key, {}) + if option_description.get("deprecated"): + deprecation_reason = option_description.get("deprecation_reason", "") + warnings.warn( + f"The '{key}' option in section {section} is deprecated. {deprecation_reason}", + DeprecationWarning, + stacklevel=2 + _extra_stacklevel, + ) # For when we rename whole sections if section in self.inversed_deprecated_sections: deprecated_section, deprecated_key = (section, key) @@ -995,7 +1003,6 @@ class AirflowConfigParser(ConfigParser): deprecated_section, deprecated_key, _ = self.deprecated_options.get( (section, key), (None, None, None) ) - # first check environment variables option = self._get_environment_variables( deprecated_key, diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 8bf4457c2f..185687ad56 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -748,7 +748,6 @@ def initdb(session: Session = NEW_SESSION, load_connections: bool = True): upgradedb(session=session) else: _create_db_from_orm(session=session) - # Load default connections if conf.getboolean("database", "LOAD_DEFAULT_CONNECTIONS") and load_connections: create_default_connections(session=session) # Add default pool & sync log_template diff --git a/dev/perf/sql_queries.py b/dev/perf/sql_queries.py index 2bddc349ef..76b87420eb 100644 --- a/dev/perf/sql_queries.py +++ b/dev/perf/sql_queries.py @@ -30,7 +30,6 @@ DAG_FOLDER = os.path.join(os.path.dirname(__file__), "dags") os.environ["AIRFLOW__CORE__DAGS_FOLDER"] = DAG_FOLDER os.environ["AIRFLOW__DEBUG__SQLALCHEMY_STATS"] = "True" os.environ["AIRFLOW__CORE__LOAD_EXAMPLES"] = "False" -os.environ["AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS"] = "True" # Here we setup simpler logger to avoid any code changes in # Airflow core code base diff --git a/docs/apache-airflow-providers-amazon/connections/aws.rst b/docs/apache-airflow-providers-amazon/connections/aws.rst index 03c815f072..ba5fe01723 100644 --- a/docs/apache-airflow-providers-amazon/connections/aws.rst +++ b/docs/apache-airflow-providers-amazon/connections/aws.rst @@ -57,7 +57,7 @@ automatically the credentials from there. This is no longer the case and the region needs to be set manually, either in the connection screens in Airflow, or via the ``AWS_DEFAULT_REGION`` environment variable. -.. caution:: If you do not set ``[database] load_default_connections`` to ``True`` +.. caution:: If you do not run "airflow connections create-default-connections" command, most probably you do not have ``aws_default``. For historical reasons, the Amazon Provider components (Hooks, Operators, Sensors, etc.) fallback to the default boto3 credentials strategy in case of a missing Connection ID. This behaviour is deprecated and will be removed in a future releases. diff --git a/docs/exts/includes/sections-and-options.rst b/docs/exts/includes/sections-and-options.rst index 497223fcee..33ea64231f 100644 --- a/docs/exts/includes/sections-and-options.rst +++ b/docs/exts/includes/sections-and-options.rst @@ -37,41 +37,44 @@ {% endif %} {% for option_name, option in section["options"].items() %} - .. _config:{{ section_name }}__{{ option_name }}: {{ option_name }} {{ "-" * option_name|length }} - {% if option["version_added"] %} - .. versionadded:: {{ option["version_added"] }} - {% endif %} + {% if option["version_added"] %} + .. versionadded:: {{ option["version_added"] }} + {% endif %} - {% if option["description"] %} + {% if option["description"] %} {{ option["description"] }} - {% endif %} + {% endif %} + + {% if option.get("version_deprecated") %} + .. deprecated:: {{ option["version_deprecated"] }} + {{ option["deprecation_reason"] | indent(width=8) }} + {% endif %} - {% if option.get("see_also") %} + {% if option.get("see_also") %} .. seealso:: {{ option["see_also"] }} - {% endif %} + {% endif %} :Type: {{ option["type"] }} :Default: ``{{ "''" if option["default"] == "" else option["default"] }}`` - {% if option.get("sensitive") %} + {% if option.get("sensitive") %} :Environment Variables: ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}`` ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_CMD`` ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_SECRET`` - {% else %} + {% else %} :Environment Variable: ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}`` - {% endif %} - {% if option["example"] %} + {% endif %} + {% if option["example"] %} :Example: ``{{ option["example"] }}`` - {% endif %} - + {% endif %} {% endfor %} {% if section_name in deprecated_options %} diff --git a/docs/helm-chart/airflow-configuration.rst b/docs/helm-chart/airflow-configuration.rst index 90ccd7d370..97aae726c1 100644 --- a/docs/helm-chart/airflow-configuration.rst +++ b/docs/helm-chart/airflow-configuration.rst @@ -39,12 +39,3 @@ configuration prior to installing and deploying the service. The recommended way to load example DAGs using the official Docker image and chart is to configure the ``AIRFLOW__CORE__LOAD_EXAMPLES`` environment variable in ``extraEnv`` (see :doc:`Parameters reference <parameters-ref>`). The official Docker image has ``AIRFLOW__CORE__LOAD_EXAMPLES=False`` set within the image, so you need to override it with an environment variable when deploying the chart in order for the examples to be present. - -.. note:: - - The ``AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS`` variable is not used by the Chart. Airflow Helm Chart is - intended to be used as production deployment and loading default connections is not supposed to be handled - during Chart installation. The Chart is intended to install and configure the Apache Airflow software - and create database structure, but not to fill-in the data which should be managed by the users. - The default connections are only meaningful when you want to have a ``quick start`` with Airflow or - do some development and adding the data via Helm Chart installation is not a good idea. diff --git a/newsfragments/33136.significant.rst b/newsfragments/33136.significant.rst new file mode 100644 index 0000000000..9f7cb3aa92 --- /dev/null +++ b/newsfragments/33136.significant.rst @@ -0,0 +1,5 @@ +The "airflow db init", "airflow db upgrade" commands and "[database] load_default_connections" configuration options are deprecated. + +Instead, you should use "airflow db migrate" command to create or upgrade database. This command will not create default connections. +In order to create default connections you need to run "airflow connections create-default-connections" explicitly, +after running "airflow db migrate". diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index e4cf2b989c..73b8a20deb 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -316,7 +316,6 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then cd "${AIRFLOW_SOURCES}" if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then - export AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} wait_for_asset_compilation # shellcheck source=scripts/in_container/bin/run_tmux diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh index b23dc18dd2..67d4f1814e 100755 --- a/scripts/in_container/check_environment.sh +++ b/scripts/in_container/check_environment.sh @@ -110,12 +110,23 @@ function startairflow_if_requested() { echo echo "Starting Airflow" echo - export AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} . "$( dirname "${BASH_SOURCE[0]}" )/configure_environment.sh" - airflow db migrate + if airflow db migrate + then + if [[ ${LOAD_DEFAULT_CONNECTIONS=} == "true" || ${LOAD_DEFAULT_CONNECTIONS=} == "True" ]]; then + echo + echo "${COLOR_BLUE}Creating default connections${COLOR_RESET}" + echo + airflow connections create-default-connections + fi + else + # For Airflow versions that do not support db migrate, we should run airflow db init and + # set the removed AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS + AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} airflow db init + fi airflow users create -u admin -p admin -f Thor -l Adminstra -r Admin -e [email protected] . "$( dirname "${BASH_SOURCE[0]}" )/run_init_script.sh" diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index a6229dc07c..3a24d33111 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -157,7 +157,6 @@ class TestConf: # test display_source cfg_dict = conf.as_dict(display_source=True) assert cfg_dict["core"]["load_examples"][1] == "airflow.cfg" - assert cfg_dict["database"]["load_default_connections"][1] == "airflow.cfg" assert cfg_dict["testsection"]["testkey"] == ("testvalue", "env var") assert cfg_dict["core"]["fernet_key"] == ("< hidden >", "env var")
