This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8c013b6a89d Rename pre-2-7 defaults to
provider_config_fallback_defaults (#44895)
8c013b6a89d is described below
commit 8c013b6a89da81832270841dea529f86cbe42c6f
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Dec 12 23:19:49 2024 +0100
Rename pre-2-7 defaults to provider_config_fallback_defaults (#44895)
* Rename pre-2-7 defaults to provider_config_fallback_defaults
After unsuccessful atempts to remove them, renaming and adding
appropriate warning against removing them seems to be a good idea.
* Update airflow/config_templates/provider_config_fallback_defaults.cfg
Co-authored-by: Jed Cunningham
<[email protected]>
---------
Co-authored-by: Jed Cunningham
<[email protected]>
---
...s.cfg => provider_config_fallback_defaults.cfg} | 45 ++++++++++++++++++----
airflow/configuration.py | 38 +++++++++++-------
2 files changed, 61 insertions(+), 22 deletions(-)
diff --git a/airflow/config_templates/pre_2_7_defaults.cfg
b/airflow/config_templates/provider_config_fallback_defaults.cfg
similarity index 66%
rename from airflow/config_templates/pre_2_7_defaults.cfg
rename to airflow/config_templates/provider_config_fallback_defaults.cfg
index 2568d42445f..ba92feaef47 100644
--- a/airflow/config_templates/pre_2_7_defaults.cfg
+++ b/airflow/config_templates/provider_config_fallback_defaults.cfg
@@ -16,15 +16,21 @@
# specific language governing permissions and limitations
# under the License.
-# This file contains pre Airflow 2.7, provider defaults for Airflow
configuration.
-# They are provided as fallback option to older version of the
-# providers that might expect them to be present.
+# This file contains provider defaults for Airflow configuration, containing
fallback default values
+# that might be needed when provider classes are being imported - before
provider's configuration
+# is loaded.
+#
+# Unfortunately airflow currently performs a lot of stuff during importing and
some of that might lead
+# to retrieving provider configuration before the defaults for the provider
are loaded.
+#
+# Those are only defaults, so if you have "real" values configured in your
configuration (.cfg file or
+# environment variables) those will be used as usual.
+
+# NOTE!! Do NOT attempt to remove those default fallbacks thinking that they
are unnecessary duplication,
+# at least not until we fix the way how airflow imports "do stuff". This is
unlikely to succeed.
+#
+# You've been warned!
#
-# NOTE !!!! Please DO NOT modify values in the file even if they change in
corresponding
-# providers. The values here should be treated as "read only" and should not
be modified
-# even if defaults in newer versions of corresponding Providers change.
-# They are only here so that backwards compatible behaviour for old provider
-# versions can be maintained.
[atlas]
sasl_enabled = False
@@ -81,6 +87,29 @@ index_patterns = _all
use_ssl = False
verify_certs = True
+[opensearch]
+host =
+port =
+username =
+password =
+log_id_template = {dag_id}-{task_id}-{run_id}-{map_index}-{try_number}
+end_of_log_mark = end_of_log
+write_stdout = False
+json_format = False
+json_fields = asctime, filename, lineno, levelname, message
+host_field = host
+offset_field = offset
+index_patterns = _all
+index_patterns_callable =
+
+[opensearch_configs]
+http_compress = False
+use_ssl = False
+verify_certs = False
+ssl_assert_hostname = False
+ssl_show_warn = False
+ca_certs =
+
[kubernetes_executor]
api_client_retry_configuration =
logs_task_metadata = False
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 76e2690ffe1..331787e7a68 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -209,7 +209,7 @@ class AirflowConfigParser(ConfigParser):
# interpolation placeholders. The _default_values config parser will
interpolate them
# properly when we call get() on it.
self._default_values =
create_default_config_parser(self.configuration_description)
- self._pre_2_7_default_values = create_pre_2_7_defaults()
+ self._provider_config_fallback_default_values =
create_provider_config_fallback_defaults()
if default_config is not None:
self._update_defaults_from_string(default_config)
self._update_logging_deprecated_template_to_one_from_defaults()
@@ -292,9 +292,9 @@ class AirflowConfigParser(ConfigParser):
return value.replace("%", "%%")
return value
- def get_default_pre_2_7_value(self, section: str, key: str, **kwargs) ->
Any:
- """Get pre 2.7 default config values."""
- return self._pre_2_7_default_values.get(section, key, fallback=None,
**kwargs)
+ def get_provider_config_fallback_defaults(self, section: str, key: str,
**kwargs) -> Any:
+ """Get provider config fallback default values."""
+ return self._provider_config_fallback_default_values.get(section, key,
fallback=None, **kwargs)
# These configuration elements can be fetched as the stdout of commands
# following the "{section}__{name}_cmd" pattern, the idea behind this
@@ -989,9 +989,9 @@ class AirflowConfigParser(ConfigParser):
if self.get_default_value(section, key) is not None or "fallback" in
kwargs:
return expand_env_var(self.get_default_value(section, key,
**kwargs))
- if self.get_default_pre_2_7_value(section, key) is not None:
+ if self.get_provider_config_fallback_defaults(section, key) is not
None:
# no expansion needed
- return self.get_default_pre_2_7_value(section, key, **kwargs)
+ return self.get_provider_config_fallback_defaults(section, key,
**kwargs)
if not suppress_warnings:
log.warning("section/key [%s/%s] not found in config", section,
key)
@@ -1382,7 +1382,7 @@ class AirflowConfigParser(ConfigParser):
# We check sequentially all those sources and the last one we saw it
in will "win"
configs: Iterable[tuple[str, ConfigParser]] = [
- ("default-pre-2-7", self._pre_2_7_default_values),
+ ("provider-fallback-defaults",
self._provider_config_fallback_default_values),
("default", self._default_values),
("airflow.cfg", self),
]
@@ -1901,17 +1901,27 @@ def
create_default_config_parser(configuration_description: dict[str, dict[str,
return parser
-def create_pre_2_7_defaults() -> ConfigParser:
+def create_provider_config_fallback_defaults() -> ConfigParser:
"""
- Create parser using the old defaults from Airflow < 2.7.0.
+ Create fallback defaults.
- This is used in order to be able to fall-back to those defaults when old
version of provider,
- not supporting "config contribution" is installed with Airflow 2.7.0+.
This "default"
- configuration does not support variable expansion, those are pretty much
hard-coded defaults '
- we want to fall-back to in such case.
+ This parser contains provider defaults for Airflow configuration,
containing fallback default values
+ that might be needed when provider classes are being imported - before
provider's configuration
+ is loaded.
+
+ Unfortunately airflow currently performs a lot of stuff during importing
and some of that might lead
+ to retrieving provider configuration before the defaults for the provider
are loaded.
+
+ Those are only defaults, so if you have "real" values configured in your
configuration (.cfg file or
+ environment variables) those will be used as usual.
+
+ NOTE!! Do NOT attempt to remove those default fallbacks thinking that they
are unnecessary duplication,
+ at least not until we fix the way how airflow imports "do stuff". This is
unlikely to succeed.
+
+ You've been warned!
"""
config_parser = ConfigParser()
- config_parser.read(_default_config_file_path("pre_2_7_defaults.cfg"))
+
config_parser.read(_default_config_file_path("provider_config_fallback_defaults.cfg"))
return config_parser
