jedcunningham commented on code in PR #32604:
URL: https://github.com/apache/airflow/pull/32604#discussion_r1269955978
##########
airflow/configuration.py:
##########
@@ -1644,6 +1765,61 @@ def remove_all_read_configurations(self):
for section in self.sections():
self.remove_section(section)
+ @property
+ def providers_configuration_loaded(self) -> bool:
+ """Checks if providers have been loaded."""
+ return self._providers_configuration_loaded
+
+ def load_providers_configuration(self):
+ """
+ Loads configuration for providers.
+
+ This should be done after initial configuration have been performed.
Initializing and discovering
+ providers is an expensive operation and cannot be performed when we
load configuration for the first
+ time when airflow starts, because we initialize configuration very
early, during importing of the
+ `airflow` package and the module is not yet ready to be used when it
happens and until configuration
+ and settings are loaded. Therefore, in order to reload provider
configuration we need to additionally
+ load provider - specific configuration.
+ """
+ log.debug("Loading providers configuration")
+ from airflow.providers_manager import ProvidersManager
+
+ for provider, config in
ProvidersManager().already_initialized_provider_configs:
+ for provider_section, provider_section_content in config.items():
+ provider_options = provider_section_content["options"]
+ section_in_current_config =
self.configuration_description.get(provider_section)
+ if not section_in_current_config:
+ # merge whole sections into existing config
+ self.configuration_description[provider_section] =
deepcopy(provider_section_content)
+ section_in_current_config =
self.configuration_description.get(provider_section)
+ for option in provider_options:
+ section_in_current_config["options"][option]["source"]
= f"default-{provider}"
+ else:
+ for option in provider_options:
+ option_in_current_config =
section_in_current_config["options"].get(option)
+ if not option_in_current_config:
+ # merge new options into existing sections
+
self.configuration_description[provider_section]["options"][option] = deepcopy(
+ provider_options[option]
+ )
+ option_in_current_config =
section_in_current_config["options"].get(option)
+ option_in_current_config["source"] =
f"default-{provider}"
+ else:
+ # replace default values for existing options
Review Comment:
Yeah, this feels wrong to me.
What if more than 1 provider decided to override a certain value?
I'd rather we raise here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
