potiuk commented on code in PR #32604:
URL: https://github.com/apache/airflow/pull/32604#discussion_r1266173257
##########
airflow/configuration.py:
##########
@@ -1474,84 +1679,123 @@ def _generate_fernet_key() -> str:
return Fernet.generate_key().decode()
-def initialize_config() -> AirflowConfigParser:
+def _create_values_from_config_description(
+ default_config_description: dict[str, dict[str, Any]]
+) -> dict[str, dict[str, Any]]:
"""
- Load the Airflow config files.
+ Creates dictionary of default values from configuration description, using
+ based on variables defined in this module (expansion variables).
- Called for you automatically as part of the Airflow boot process.
+ :param default_config_description: description of configuration to read
including defaults
+ :return: dictionary of default values based on description and variables
defined in this module
"""
- global FERNET_KEY, AIRFLOW_HOME, WEBSERVER_CONFIG
-
- default_config = _parameterized_config_from_template("default_airflow.cfg")
+ all_vars = get_all_expansion_variables()
+ default_values: dict[str, dict[str, Any]] = {}
+ for section, section_desc in default_config_description.items():
+ section_dict: dict[str, Any] = {}
+ default_values[section] = section_dict
+ options = section_desc["options"]
+ for key in options:
+ default_value = options[key]["default"]
+ if default_value is not None:
+ section_dict[key] = (
+ default_value.format(**all_vars) if
isinstance(default_value, str) else default_value
+ )
+ return default_values
- local_conf = AirflowConfigParser(default_config=default_config)
- if local_conf.getboolean("core", "unit_test_mode"):
- # Load test config only
- if not os.path.isfile(TEST_CONFIG_FILE):
- from cryptography.fernet import Fernet
+def create_airflow_config_parser(default_config_description: dict[str,
dict[str, Any]]):
+ """
+ Creates Airflow config parser.
- log.info("Creating new Airflow config file for unit tests in: %s",
TEST_CONFIG_FILE)
- pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True)
+ Creates Airflow config parser based on variables defined in this module
(expansion variables)
+ and default configuration description passed as parameter which serves as
source of default values.
- FERNET_KEY = Fernet.generate_key().decode()
+ :param default_config_description: default configuration description -
retrieved from config*.yaml files
+ following the schema defined in "config.yml.schema.json" in the
config_templates folder.
+ :return: Airflow config parser that can be used to read configuration
values from.
+ """
+ return AirflowConfigParser(
+
default_config=_create_values_from_config_description(default_config_description),
+ default_config_description=default_config_description,
+ )
- with open(TEST_CONFIG_FILE, "w") as file:
- cfg = _parameterized_config_from_template("default_test.cfg")
- file.write(cfg)
- make_group_other_inaccessible(TEST_CONFIG_FILE)
- local_conf.load_test_config()
- else:
- # Load normal config
- if not os.path.isfile(AIRFLOW_CONFIG):
- from cryptography.fernet import Fernet
+def load_standard_airflow_configuration(airflow_config_parser:
AirflowConfigParser):
+ """
+ Loads standard airflow configuration.
- log.info("Creating new Airflow config file in: %s", AIRFLOW_CONFIG)
- pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True)
+ In case it finds that the configuration file is missing, it will create it
and write the default
+ configuration values there, based on defaults passed, and will add the
comments and examples
+ from the default configuration description passed.
- FERNET_KEY = Fernet.generate_key().decode()
+ :param airflow_config_parser: parser to which the configuration will be
loaded
+ """
+ global FERNET_KEY, AIRFLOW_HOME
+ if not os.path.isfile(AIRFLOW_CONFIG):
+ from cryptography.fernet import Fernet
+
+ log.info("Creating new Airflow config file in: %s", AIRFLOW_CONFIG)
+ pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True)
+ FERNET_KEY = Fernet.generate_key().decode()
+ if airflow_config_parser._default_values and
airflow_config_parser.default_config_description:
with open(AIRFLOW_CONFIG, "w") as file:
- file.write(default_config)
- make_group_other_inaccessible(AIRFLOW_CONFIG)
-
- log.info("Reading the config from %s", AIRFLOW_CONFIG)
+ airflow_config_parser.write(file)
+ make_group_other_inaccessible(AIRFLOW_CONFIG)
+ log.info("Reading the config from %s", AIRFLOW_CONFIG)
+ airflow_config_parser.read(AIRFLOW_CONFIG)
+ if airflow_config_parser.has_option("core", "AIRFLOW_HOME"):
+ msg = (
+ "Specifying both AIRFLOW_HOME environment variable and
airflow_home "
+ "in the config file is deprecated. Please use only the
AIRFLOW_HOME "
+ "environment variable and remove the config file entry."
+ )
+ if "AIRFLOW_HOME" in os.environ:
+ warnings.warn(msg, category=DeprecationWarning)
+ elif airflow_config_parser.get("core", "airflow_home") == AIRFLOW_HOME:
+ warnings.warn(
+ "Specifying airflow_home in the config file is deprecated. As
you "
+ "have left it at the default value you should remove the
setting "
+ "from your airflow.cfg and suffer no change in behaviour.",
+ category=DeprecationWarning,
+ )
+ else:
+ # there
+ AIRFLOW_HOME = airflow_config_parser.get("core", "airflow_home")
# type: ignore[assignment]
+ warnings.warn(msg, category=DeprecationWarning)
- local_conf.read(AIRFLOW_CONFIG)
- if local_conf.has_option("core", "AIRFLOW_HOME"):
- msg = (
- "Specifying both AIRFLOW_HOME environment variable and
airflow_home "
- "in the config file is deprecated. Please use only the
AIRFLOW_HOME "
- "environment variable and remove the config file entry."
- )
- if "AIRFLOW_HOME" in os.environ:
- warnings.warn(msg, category=DeprecationWarning)
- elif local_conf.get("core", "airflow_home") == AIRFLOW_HOME:
- warnings.warn(
- "Specifying airflow_home in the config file is deprecated.
As you "
- "have left it at the default value you should remove the
setting "
- "from your airflow.cfg and suffer no change in behaviour.",
- category=DeprecationWarning,
- )
- else:
- # there
- AIRFLOW_HOME = local_conf.get("core", "airflow_home") # type:
ignore[assignment]
- warnings.warn(msg, category=DeprecationWarning)
+def initialize_config() -> AirflowConfigParser:
+ """
+ Load the Airflow config files.
- # They _might_ have set unit_test_mode in the airflow.cfg, we still
- # want to respect that and then load the unittests.cfg
- if local_conf.getboolean("core", "unit_test_mode"):
- local_conf.load_test_config()
+ Called for you automatically as part of the Airflow boot process.
+ """
+ global FERNET_KEY, AIRFLOW_HOME, WEBSERVER_CONFIG
Review Comment:
They are actually used by
```
def get_all_expansion_variables() -> dict[str, Any]:
return {k: v for d in [globals(), locals()] for k, v in d.items()}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
