This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 33222eaeeb5 Drop provider dependency from airflow-core secrets backend
tests (#71677)
33222eaeeb5 is described below
commit 33222eaeeb57be071be992e9ce5a25d2c3a56b2e
Author: rjgoyln <[email protected]>
AuthorDate: Sat Aug 29 01:47:00 2026 +0800
Drop provider dependency from airflow-core secrets backend tests (#71677)
These tests cover how core loads and configures a custom secrets backend,
not how any particular backend behaves. Pinning them to the amazon and
hashicorp providers made a core-only environment unable to run them, and
kept them skipped whenever the full provider set is not installed.
---
airflow-core/tests/unit/always/test_secrets.py | 98 ++++++--------------
airflow-core/tests/unit/core/test_configuration.py | 102 ++++++---------------
.../test_utils/fake_secrets_backend.py | 59 ++++++++++++
3 files changed, 114 insertions(+), 145 deletions(-)
diff --git a/airflow-core/tests/unit/always/test_secrets.py
b/airflow-core/tests/unit/always/test_secrets.py
index 9372cb08a83..4d7371303b6 100644
--- a/airflow-core/tests/unit/always/test_secrets.py
+++ b/airflow-core/tests/unit/always/test_secrets.py
@@ -29,10 +29,9 @@ from airflow.secrets import BaseSecretsBackend
from tests_common.test_utils.config import conf_vars
from tests_common.test_utils.db import clear_db_variables
-from tests_common.test_utils.markers import
skip_if_force_lowest_dependencies_marker
+from tests_common.test_utils.fake_secrets_backend import FAKE_BACKEND_PATH,
FakeSecretsBackend
-@skip_if_force_lowest_dependencies_marker
class TestConnectionsFromSecrets:
def setup_method(self) -> None:
SecretCache.reset()
@@ -55,11 +54,8 @@ class TestConnectionsFromSecrets:
@conf_vars(
{
- (
- "secrets",
- "backend",
- ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
- ("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow",
"profile_name": null}',
+ ("secrets", "backend"): FAKE_BACKEND_PATH,
+ ("secrets", "backend_kwargs"): '{"connections_prefix":
"/airflow"}',
}
)
def test_initialize_secrets_backends(self):
@@ -67,34 +63,26 @@ class TestConnectionsFromSecrets:
backend_classes = [backend.__class__.__name__ for backend in backends]
assert len(backends) == 3
- assert "SystemsManagerParameterStoreBackend" in backend_classes
+ assert "FakeSecretsBackend" in backend_classes
@conf_vars(
{
- (
- "secrets",
- "backend",
- ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ ("secrets", "backend"): FAKE_BACKEND_PATH,
("secrets", "backend_kwargs"): '{"use_ssl": false}',
}
)
def test_backends_kwargs(self):
backends = initialize_secrets_backends()
- systems_manager = next(
- backend
- for backend in backends
- if backend.__class__.__name__ ==
"SystemsManagerParameterStoreBackend"
+ fake_backend = next(
+ backend for backend in backends if backend.__class__.__name__ ==
"FakeSecretsBackend"
)
- assert systems_manager.kwargs == {}
- assert systems_manager.use_ssl is False
+ assert fake_backend.kwargs == {}
+ assert fake_backend.use_ssl is False
@conf_vars(
{
- (
- "secrets",
- "backend",
- ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
- ("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow",
"profile_name": null}',
+ ("secrets", "backend"): FAKE_BACKEND_PATH,
+ ("secrets", "backend_kwargs"): '{"connections_prefix":
"/airflow"}',
}
)
@mock.patch.dict(
@@ -103,20 +91,16 @@ class TestConnectionsFromSecrets:
"AIRFLOW_CONN_TEST_MYSQL":
"mysql://airflow:airflow@host:5432/airflow",
},
)
- @mock.patch(
- "airflow.providers.amazon.aws.secrets.systems_manager."
- "SystemsManagerParameterStoreBackend.get_connection"
- )
+ @mock.patch.object(FakeSecretsBackend, "get_connection")
def test_backend_fallback_to_env_var(self, mock_get_connection):
mock_get_connection.return_value = None
backends = ensure_secrets_loaded()
backend_classes = [backend.__class__.__name__ for backend in backends]
- assert "SystemsManagerParameterStoreBackend" in backend_classes
+ assert "FakeSecretsBackend" in backend_classes
conn = Connection.get_connection_from_secrets(conn_id="test_mysql")
- # Assert that SystemsManagerParameterStoreBackend.get_conn_uri was
called
mock_get_connection.assert_called_once_with(conn_id="test_mysql",
team_name=None)
assert conn.get_uri() == "mysql://airflow:airflow@host:5432/airflow"
@@ -134,7 +118,6 @@ class TestConnectionsFromSecrets:
Connection.get_connection_from_secrets(conn_id="_team___test_mysql")
-@skip_if_force_lowest_dependencies_marker
@pytest.mark.db_test
class TestVariableFromSecrets:
def setup_method(self) -> None:
@@ -181,11 +164,8 @@ class TestVariableFromSecrets:
@conf_vars(
{
- (
- "secrets",
- "backend",
- ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
- ("secrets", "backend_kwargs"): '{"variables_prefix": "/airflow",
"profile_name": null}',
+ ("secrets", "backend"): FAKE_BACKEND_PATH,
+ ("secrets", "backend_kwargs"): '{"variables_prefix": "/airflow"}',
}
)
@mock.patch.dict(
@@ -195,14 +175,11 @@ class TestVariableFromSecrets:
},
)
@mock.patch("airflow.secrets.metastore.MetastoreBackend.get_variable")
- @mock.patch(
- "airflow.providers.amazon.aws.secrets.systems_manager."
- "SystemsManagerParameterStoreBackend.get_variable"
- )
+ @mock.patch.object(FakeSecretsBackend, "get_variable")
def test_backend_variable_order(self, mock_secret_get, mock_meta_get):
backends = ensure_secrets_loaded()
backend_classes = [backend.__class__.__name__ for backend in backends]
- assert "SystemsManagerParameterStoreBackend" in backend_classes
+ assert "FakeSecretsBackend" in backend_classes
mock_secret_get.return_value = None
mock_meta_get.return_value = None
@@ -250,7 +227,6 @@ class _TeamUnawareGetVariableBackend(BaseSecretsBackend):
return self._vars.get(key)
-@skip_if_force_lowest_dependencies_marker
class TestTeamUnawareBackendSignatureCompat:
"""Backends whose overrides predate the ``team_name`` keyword must keep
working (issue #1333)."""
@@ -279,21 +255,13 @@ class TestTeamUnawareBackendSignatureCompat:
assert value == "secret_value"
-@skip_if_force_lowest_dependencies_marker
class TestSecretBackendKwargEnvVars:
"""Test per-key env var overrides for secrets backend kwargs."""
def setup_method(self) -> None:
SecretCache.reset()
- @conf_vars(
- {
- (
- "secrets",
- "backend",
- ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
- }
- )
+ @conf_vars({("secrets", "backend"): FAKE_BACKEND_PATH})
@mock.patch.dict(
"os.environ",
{"AIRFLOW__SECRETS__BACKEND_KWARG__CONNECTIONS_PREFIX":
"/airflow/connections"},
@@ -301,17 +269,12 @@ class TestSecretBackendKwargEnvVars:
def test_backend_kwarg_env_vars_basic(self):
"""Per-key env var is picked up when no JSON blob is set."""
backends = initialize_secrets_backends()
- systems_manager = next(
- b for b in backends if b.__class__.__name__ ==
"SystemsManagerParameterStoreBackend"
- )
- assert systems_manager.connections_prefix == "/airflow/connections"
+ fake_backend = next(b for b in backends if b.__class__.__name__ ==
"FakeSecretsBackend")
+ assert fake_backend.connections_prefix == "/airflow/connections"
@conf_vars(
{
- (
- "secrets",
- "backend",
- ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ ("secrets", "backend"): FAKE_BACKEND_PATH,
("secrets", "backend_kwargs"): '{"connections_prefix": "/old"}',
}
)
@@ -322,17 +285,12 @@ class TestSecretBackendKwargEnvVars:
def test_backend_kwarg_env_vars_override_json(self):
"""Per-key env var overrides the same key in the JSON blob."""
backends = initialize_secrets_backends()
- systems_manager = next(
- b for b in backends if b.__class__.__name__ ==
"SystemsManagerParameterStoreBackend"
- )
- assert systems_manager.connections_prefix == "/new"
+ fake_backend = next(b for b in backends if b.__class__.__name__ ==
"FakeSecretsBackend")
+ assert fake_backend.connections_prefix == "/new"
@conf_vars(
{
- (
- "secrets",
- "backend",
- ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ ("secrets", "backend"): FAKE_BACKEND_PATH,
("secrets", "backend_kwargs"): '{"connections_prefix":
"/airflow"}',
}
)
@@ -343,8 +301,6 @@ class TestSecretBackendKwargEnvVars:
def test_backend_kwarg_env_vars_merge_with_json(self):
"""Per-key env var is merged with (not replacing) the JSON blob."""
backends = initialize_secrets_backends()
- systems_manager = next(
- b for b in backends if b.__class__.__name__ ==
"SystemsManagerParameterStoreBackend"
- )
- assert systems_manager.connections_prefix == "/airflow"
- assert systems_manager.variables_prefix == "/airflow/variables"
+ fake_backend = next(b for b in backends if b.__class__.__name__ ==
"FakeSecretsBackend")
+ assert fake_backend.connections_prefix == "/airflow"
+ assert fake_backend.variables_prefix == "/airflow/variables"
diff --git a/airflow-core/tests/unit/core/test_configuration.py
b/airflow-core/tests/unit/core/test_configuration.py
index 815574a30e1..e173215234a 100644
--- a/airflow-core/tests/unit/core/test_configuration.py
+++ b/airflow-core/tests/unit/core/test_configuration.py
@@ -52,6 +52,11 @@ from tests_common.test_utils.config import (
conf_vars,
create_fresh_airflow_config,
)
+from tests_common.test_utils.fake_secrets_backend import (
+ FAKE_BACKEND_PATH,
+ FAKE_CONFIG_BACKEND_PATH,
+ FAKE_UNREACHABLE_BACKEND_PATH,
+)
from tests_common.test_utils.markers import
skip_if_force_lowest_dependencies_marker
from tests_common.test_utils.reset_warning_registry import
reset_warning_registry
from unit.utils.test_config import (
@@ -421,37 +426,16 @@ class TestConf:
):
assert test_conf.getboolean(section, key) is False
- @skip_if_force_lowest_dependencies_marker
-
@mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac")
@conf_vars(
{
- ("secrets", "backend"):
"airflow.providers.hashicorp.secrets.vault.VaultBackend",
- ("secrets", "backend_kwargs"): '{"url": "http://127.0.0.1:8200";,
"token": "token"}',
+ ("secrets", "backend"): FAKE_CONFIG_BACKEND_PATH,
+ ("secrets", "backend_kwargs"): (
+ '{"config_values": {"sql_alchemy_conn":
"sqlite:////Users/airflow/airflow/airflow.db"}}'
+ ),
}
)
- def test_config_from_secret_backend(self, mock_hvac):
+ def test_config_from_secret_backend(self):
"""Get Config Value from a Secret Backend"""
- mock_client = mock.MagicMock()
- mock_hvac.Client.return_value = mock_client
- mock_client.secrets.kv.v2.read_secret_version.return_value = {
- "request_id": "2d48a2ad-6bcb-e5b6-429d-da35fdf31f56",
- "lease_id": "",
- "renewable": False,
- "lease_duration": 0,
- "data": {
- "data": {"value":
"sqlite:////Users/airflow/airflow/airflow.db"},
- "metadata": {
- "created_time": "2020-03-28T02:10:54.301784Z",
- "deletion_time": "",
- "destroyed": False,
- "version": 1,
- },
- },
- "wrap_info": None,
- "warnings": None,
- "auth": None,
- }
-
test_config = textwrap.dedent(
"""
[test]
@@ -507,21 +491,9 @@ class TestConf:
# is True
assert test_conf.as_dict(display_sensitive=True, include_cmds=False)
- @skip_if_force_lowest_dependencies_marker
-
@mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac")
- @conf_vars(
- {
- ("secrets", "backend"):
"airflow.providers.hashicorp.secrets.vault.VaultBackend",
- ("secrets", "backend_kwargs"): '{"url": "http://127.0.0.1:8200";,
"token": "token"}',
- }
- )
- def test_config_raise_exception_from_secret_backend_connection_error(self,
mock_hvac):
+ @conf_vars({("secrets", "backend"): FAKE_UNREACHABLE_BACKEND_PATH})
+ def test_config_raise_exception_from_secret_backend_connection_error(self):
"""Get Config Value from a Secret Backend"""
- mock_client = mock.MagicMock()
- # mock_client.side_effect = AirflowConfigException
- mock_hvac.Client.return_value = mock_client
- mock_client.secrets.kv.v2.read_secret_version.return_value = Exception
-
test_config = textwrap.dedent(
"""
[test]
@@ -815,14 +787,10 @@ class TestConf:
assert conf_materialize_cmds["database"]["sql_alchemy_conn"] ==
"postgresql+psycopg2://"
- @skip_if_force_lowest_dependencies_marker
@conf_vars(
{
- (
- "workers",
- "secrets_backend",
- ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
- ("workers", "secrets_backend_kwargs"): '{"connections_prefix":
"/airflow", "profile_name": null}',
+ ("workers", "secrets_backend"): FAKE_BACKEND_PATH,
+ ("workers", "secrets_backend_kwargs"): '{"connections_prefix":
"/airflow"}',
}
)
def test_initialize_secrets_backends_on_workers(self):
@@ -831,32 +799,25 @@ class TestConf:
backend_classes = [backend.__class__.__name__ for backend in backends]
assert len(backends) == 3
- assert "SystemsManagerParameterStoreBackend" in backend_classes
+ assert "FakeSecretsBackend" in backend_classes
assert "EnvironmentVariablesBackend" in backend_classes
assert "ExecutionAPISecretsBackend" in backend_classes
- @skip_if_force_lowest_dependencies_marker
@conf_vars(
{
- (
- "workers",
- "secrets_backend",
- ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ ("workers", "secrets_backend"): FAKE_BACKEND_PATH,
("workers", "secrets_backend_kwargs"): '{"use_ssl": false}',
}
)
def test_secrets_backends_kwargs_on_workers(self):
"""Tests if secrets backend kwargs are loaded correctly for workers."""
backends =
initialize_secrets_backends(DEFAULT_SECRETS_SEARCH_PATH_WORKERS)
- systems_manager = next(
- backend
- for backend in backends
- if backend.__class__.__name__ ==
"SystemsManagerParameterStoreBackend"
+ fake_backend = next(
+ backend for backend in backends if backend.__class__.__name__ ==
"FakeSecretsBackend"
)
- assert systems_manager.kwargs == {}
- assert systems_manager.use_ssl is False
+ assert fake_backend.kwargs == {}
+ assert fake_backend.use_ssl is False
- @skip_if_force_lowest_dependencies_marker
@pytest.mark.parametrize(
(
"secrets_backend",
@@ -869,21 +830,21 @@ class TestConf:
[
# pick right backend and kwargs
pytest.param(
-
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
- '{"connections_prefix": "/airflow", "profile_name": null}',
+ FAKE_BACKEND_PATH,
+ '{"connections_prefix": "/airflow", "variables_prefix": null}',
"airflow.secrets.local_filesystem.LocalFilesystemBackend",
'{"connections_file_path": "/files/conn.json",
"variables_file_path": "/files/var.json"}',
-
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
- {"connections_prefix": "/airflow", "profile_name": None},
+ FAKE_BACKEND_PATH,
+ {"connections_prefix": "/airflow", "variables_prefix": None},
id="both-defined",
),
# do not pick kwargs of secrets backend when not defined for worker
pytest.param(
-
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ FAKE_BACKEND_PATH,
"",
"airflow.secrets.local_filesystem.LocalFilesystemBackend",
'{"connections_file_path": "/files/conn.json",
"variables_file_path": "/files/var.json"}',
-
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ FAKE_BACKEND_PATH,
{},
id="worker-backend-defined-not-kwargs",
),
@@ -960,14 +921,7 @@ class TestConf:
result = _collect_kwarg_env_vars("AIRFLOW__SECRETS__BACKEND_KWARG__")
assert result == {"role_id": "abc"}
- @conf_vars(
- {
- (
- "workers",
- "secrets_backend",
- ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
- }
- )
+ @conf_vars({("workers", "secrets_backend"): FAKE_BACKEND_PATH})
@mock.patch.dict(
"os.environ",
{"AIRFLOW__WORKERS__SECRETS_BACKEND_KWARG__CONNECTIONS_PREFIX":
"/worker/connections"},
@@ -976,7 +930,7 @@ class TestConf:
"""Per-key env var is picked up for the workers secrets backend."""
backends = ensure_secrets_loaded(DEFAULT_SECRETS_SEARCH_PATH_WORKERS)
secrets_backend = backends[0]
- assert secrets_backend.__class__.__name__ ==
"SystemsManagerParameterStoreBackend"
+ assert secrets_backend.__class__.__name__ == "FakeSecretsBackend"
assert secrets_backend.connections_prefix == "/worker/connections"
@mock.patch("airflow._shared.secrets_masker.mask_secret")
diff --git a/devel-common/src/tests_common/test_utils/fake_secrets_backend.py
b/devel-common/src/tests_common/test_utils/fake_secrets_backend.py
new file mode 100644
index 00000000000..2980b3faf6f
--- /dev/null
+++ b/devel-common/src/tests_common/test_utils/fake_secrets_backend.py
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from airflow.secrets import BaseSecretsBackend
+
+FAKE_BACKEND_PATH = f"{__name__}.FakeSecretsBackend"
+FAKE_CONFIG_BACKEND_PATH = f"{__name__}.FakeConfigSecretsBackend"
+FAKE_UNREACHABLE_BACKEND_PATH = f"{__name__}.FakeUnreachableSecretsBackend"
+
+
+class FakeSecretsBackend(BaseSecretsBackend):
+ """Secrets backend for tests that exercise backend configuration rather
than lookups."""
+
+ # Defaults deliberately differ from the values the tests pass in, so an
assertion on an
+ # attribute fails unless the configured kwarg — including an explicit JSON
null — reached here.
+ def __init__(
+ self,
+ connections_prefix: str | None = "/connections",
+ variables_prefix: str | None = "/variables",
+ use_ssl: bool = True,
+ **kwargs,
+ ) -> None:
+ self.connections_prefix = connections_prefix
+ self.variables_prefix = variables_prefix
+ self.use_ssl = use_ssl
+ self.kwargs = kwargs
+
+
+class FakeConfigSecretsBackend(BaseSecretsBackend):
+ """Secrets backend serving config values from a mapping passed as a
backend kwarg."""
+
+ def __init__(self, config_values: dict[str, str] | None = None, **kwargs)
-> None:
+ self.config_values = config_values or {}
+ self.kwargs = kwargs
+
+ def get_config(self, key: str) -> str | None:
+ return self.config_values.get(key)
+
+
+class FakeUnreachableSecretsBackend(BaseSecretsBackend):
+ """Secrets backend whose config lookups fail, standing in for an
unreachable backend."""
+
+ def get_config(self, key: str) -> str | None:
+ raise ConnectionError(f"Cannot reach the secrets backend to look up
{key!r}")
