This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6208a479924 Implement get_config in LocalFilesystemBackend (#59062)
6208a479924 is described below
commit 6208a479924d0df8ad2a47c49193fe7dc31f62e9
Author: ecodina <[email protected]>
AuthorDate: Sun Dec 7 23:59:10 2025 +0100
Implement get_config in LocalFilesystemBackend (#59062)
* implement get_config in localfilesystem secrets
* add unit tests for localfilesystem secrets
* Improve logging
Co-authored-by: Jens Scheffler <[email protected]>
---------
Co-authored-by: Jens Scheffler <[email protected]>
---
.../src/airflow/secrets/local_filesystem.py | 39 ++++++++++-
.../unit/always/test_secrets_local_filesystem.py | 75 ++++++++++++++++++++++
2 files changed, 113 insertions(+), 1 deletion(-)
diff --git a/airflow-core/src/airflow/secrets/local_filesystem.py
b/airflow-core/src/airflow/secrets/local_filesystem.py
index e3d1a98d689..efcd3b69b0c 100644
--- a/airflow-core/src/airflow/secrets/local_filesystem.py
+++ b/airflow-core/src/airflow/secrets/local_filesystem.py
@@ -278,6 +278,26 @@ def load_connections_dict(file_path: str) -> dict[str,
Any]:
return connection_by_conn_id
+def load_configs_dict(file_path: str) -> dict[str, str]:
+ """
+ Load configs from a text file.
+
+ ``JSON``, `YAML` and ``.env`` files are supported.
+
+ :param file_path: The location of the file that will be processed.
+ :return: A dictionary where the key contains a config name and the value
contains the config value.
+ """
+ log.debug("Loading configs from text file %s", file_path)
+
+ secrets = _parse_secret_file(file_path)
+ invalid_keys = [key for key, values in secrets.items() if
isinstance(values, list) and len(values) != 1]
+ if invalid_keys:
+ raise VariableNotUnique(f'The "{file_path}" file contains multiple
values for keys: {invalid_keys}')
+ configs = {key: values[0] if isinstance(values, list) else values for key,
values in secrets.items()}
+ log.debug("Loaded %d configs: ", len(configs))
+ return configs
+
+
class LocalFilesystemBackend(BaseSecretsBackend, LoggingMixin):
"""
Retrieves Connection objects and Variables from local files.
@@ -288,10 +308,16 @@ class LocalFilesystemBackend(BaseSecretsBackend,
LoggingMixin):
:param connections_file_path: File location with connection data.
"""
- def __init__(self, variables_file_path: str | None = None,
connections_file_path: str | None = None):
+ def __init__(
+ self,
+ variables_file_path: str | None = None,
+ connections_file_path: str | None = None,
+ configs_file_path: str | None = None,
+ ):
super().__init__()
self.variables_file = variables_file_path
self.connections_file = connections_file_path
+ self.configs_file = configs_file_path
@property
def _local_variables(self) -> dict[str, str]:
@@ -310,6 +336,14 @@ class LocalFilesystemBackend(BaseSecretsBackend,
LoggingMixin):
return {}
return load_connections_dict(self.connections_file)
+ @property
+ def _local_configs(self) -> dict[str, str]:
+ if not self.configs_file:
+ self.log.debug("The file for configs is not specified. Skipping")
+ # The user may not specify any file.
+ return {}
+ return load_configs_dict(self.configs_file)
+
def get_connection(self, conn_id: str) -> Connection | None:
if conn_id in self._local_connections:
return self._local_connections[conn_id]
@@ -317,3 +351,6 @@ class LocalFilesystemBackend(BaseSecretsBackend,
LoggingMixin):
def get_variable(self, key: str) -> str | None:
return self._local_variables.get(key)
+
+ def get_config(self, key: str) -> str | None:
+ return self._local_configs.get(key)
diff --git a/airflow-core/tests/unit/always/test_secrets_local_filesystem.py
b/airflow-core/tests/unit/always/test_secrets_local_filesystem.py
index 8f4cb4e12e6..fddcbf99838 100644
--- a/airflow-core/tests/unit/always/test_secrets_local_filesystem.py
+++ b/airflow-core/tests/unit/always/test_secrets_local_filesystem.py
@@ -442,6 +442,80 @@ class TestLoadConnection:
assert conn_uri_by_conn_id_yaml == conn_uri_by_conn_id_yml
+class TestLoadConfigs:
+ @pytest.mark.parametrize(
+ ("file_content", "expected_configs"),
+ [
+ ("", {}),
+ ("KEY=AAA", {"KEY": "AAA"}),
+ ("KEY_A=AAA\nKEY_B=BBB", {"KEY_A": "AAA", "KEY_B": "BBB"}),
+ ("KEY_A=AAA\n # AAAA\nKEY_B=BBB", {"KEY_A": "AAA", "KEY_B":
"BBB"}),
+ ("\n\n\n\nKEY_A=AAA\n\n\n\n\nKEY_B=BBB\n\n\n", {"KEY_A": "AAA",
"KEY_B": "BBB"}),
+ ('KEY_DICT=\'{"k1": "val1", "k2": "val2"}\'', {"KEY_DICT":
'\'{"k1": "val1", "k2": "val2"}\''}),
+ ],
+ )
+ def test_env_file_should_load_configs(self, file_content,
expected_configs):
+ with mock_local_file(file_content):
+ configs = local_filesystem.load_configs_dict("a.env")
+ assert expected_configs == configs
+
+ @pytest.mark.parametrize(
+ ("content", "expected_message"),
+ [
+ ("AA=A\nAA=B", "The \"a.env\" file contains multiple values for
keys: ['AA']"),
+ ],
+ )
+ def test_env_file_invalid_logic(self, content, expected_message):
+ with mock_local_file(content):
+ with pytest.raises(VariableNotUnique,
match=re.escape(expected_message)):
+ local_filesystem.load_configs_dict("a.env")
+
+ @pytest.mark.parametrize(
+ ("file_content", "expected_configs"),
+ [
+ ({}, {}),
+ ({"KEY": "AAA"}, {"KEY": "AAA"}),
+ ({"KEY_A": "AAA", "KEY_B": "BBB"}, {"KEY_A": "AAA", "KEY_B":
"BBB"}),
+ ],
+ )
+ def test_json_file_should_load_configs(self, file_content,
expected_configs):
+ with mock_local_file(json.dumps(file_content)):
+ configs = local_filesystem.load_configs_dict("a.json")
+ assert expected_configs == configs
+
+ def test_missing_file(self):
+ with pytest.raises(FileNotFoundError):
+ local_filesystem.load_configs_dict("a.json")
+
+ @pytest.mark.parametrize(
+ ("file_content", "expected_configs"),
+ [
+ ("KEY: AAA", {"KEY": "AAA"}),
+ (
+ """
+ KEY:
+ KEY_1:
+ - item1
+ - item2
+ """,
+ {"KEY": {"KEY_1": ["item1", "item2"]}},
+ ),
+ (
+ """
+ KEY_A: AAA
+ KEY_B: BBB
+ """,
+ {"KEY_A": "AAA", "KEY_B": "BBB"},
+ ),
+ ],
+ )
+ def test_yaml_file_should_load_configs(self, file_content,
expected_configs):
+ with mock_local_file(file_content):
+ configs_yaml = local_filesystem.load_configs_dict("a.yaml")
+ configs_yml = local_filesystem.load_configs_dict("a.yml")
+ assert expected_configs == configs_yaml == configs_yml
+
+
class TestLocalFileBackend:
def test_should_read_variable(self, tmp_path):
path = tmp_path / "testfile.var.env"
@@ -478,3 +552,4 @@ class TestLocalFileBackend:
backend = LocalFilesystemBackend()
assert backend.get_connection("CONN_A") is None
assert backend.get_variable("VAR_A") is None
+ assert backend.get_config("CONF_A") is None
