This is an automated email from the ASF dual-hosted git repository.
taragolis pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6f16c9f5a2 Disable by default redact sensitive values in tests (#38765)
6f16c9f5a2 is described below
commit 6f16c9f5a298e5b6b26bf12afa2d79b1b7eafba7
Author: Andrey Anshin <[email protected]>
AuthorDate: Sat Apr 6 12:08:29 2024 +0400
Disable by default redact sensitive values in tests (#38765)
---
contributing-docs/testing/unit_tests.rst | 19 +++++++++++++++++++
.../endpoints/test_connection_endpoint.py | 1 +
.../api_connexion/endpoints/test_dataset_endpoint.py | 1 +
.../endpoints/test_variable_endpoint.py | 1 +
tests/cli/commands/test_task_command.py | 1 +
tests/conftest.py | 20 ++++++++++++++++++++
tests/models/test_renderedtifields.py | 1 +
tests/providers/openlineage/plugins/test_utils.py | 1 +
tests/utils/log/test_secrets_masker.py | 4 +---
tests/www/views/test_views_connection.py | 1 +
tests/www/views/test_views_decorators.py | 1 +
tests/www/views/test_views_rendered.py | 2 ++
12 files changed, 50 insertions(+), 3 deletions(-)
diff --git a/contributing-docs/testing/unit_tests.rst
b/contributing-docs/testing/unit_tests.rst
index 57d2949520..00c9114c64 100644
--- a/contributing-docs/testing/unit_tests.rst
+++ b/contributing-docs/testing/unit_tests.rst
@@ -1111,6 +1111,25 @@ This prepares airflow .whl package in the dist folder.
Other Settings
--------------
+Enable masking secrets in tests
+...............................
+
+By default masking secrets in test disabled because it might have side effects
+into the other tests which intends to check logging/stdout/stderr values
+
+If you need to test masking secrets in test cases
+you have to apply ``pytest.mark.enable_redact`` to the specific test case,
class or module.
+
+
+.. code-block:: python
+
+ @pytest.mark.enable_redact
+ def test_masking(capsys):
+ mask_secret("eggs")
+ RedactedIO().write("spam eggs and potatoes")
+ assert "spam *** and potatoes" in capsys.readouterr().out
+
+
Skip test on unsupported platform / environment
...............................................
diff --git a/tests/api_connexion/endpoints/test_connection_endpoint.py
b/tests/api_connexion/endpoints/test_connection_endpoint.py
index 0d9debeb98..dc0f2893e0 100644
--- a/tests/api_connexion/endpoints/test_connection_endpoint.py
+++ b/tests/api_connexion/endpoints/test_connection_endpoint.py
@@ -144,6 +144,7 @@ class TestGetConnection(TestConnectionEndpoint):
"extra": "{'param': 'value'}",
}
+ @pytest.mark.enable_redact
def test_should_mask_sensitive_values_in_extra(self, session):
connection_model = Connection(
conn_id="test-connection-id",
diff --git a/tests/api_connexion/endpoints/test_dataset_endpoint.py
b/tests/api_connexion/endpoints/test_dataset_endpoint.py
index 29192a3c65..a2451fb30a 100644
--- a/tests/api_connexion/endpoints/test_dataset_endpoint.py
+++ b/tests/api_connexion/endpoints/test_dataset_endpoint.py
@@ -620,6 +620,7 @@ class TestPostDatasetEvents(TestDatasetEndpoint):
expected_extra=event_payload,
)
+ @pytest.mark.enable_redact
def test_should_mask_sensitive_extra_logs(self, session):
self._create_dataset(session)
event_payload = {"dataset_uri": "s3://bucket/key", "extra":
{"password": "bar"}}
diff --git a/tests/api_connexion/endpoints/test_variable_endpoint.py
b/tests/api_connexion/endpoints/test_variable_endpoint.py
index f0a1b7502b..0e300b0a8f 100644
--- a/tests/api_connexion/endpoints/test_variable_endpoint.py
+++ b/tests/api_connexion/endpoints/test_variable_endpoint.py
@@ -366,6 +366,7 @@ class TestPostVariables(TestVariableEndpoint):
"description": description,
}
+ @pytest.mark.enable_redact
def test_should_create_masked_variable(self, session):
payload = {"key": "api_key", "value": "secret_key", "description":
"secret"}
response = self.client.post(
diff --git a/tests/cli/commands/test_task_command.py
b/tests/cli/commands/test_task_command.py
index a3cc88f3d3..99c78374cc 100644
--- a/tests/cli/commands/test_task_command.py
+++ b/tests/cli/commands/test_task_command.py
@@ -153,6 +153,7 @@ class TestCliTasks:
task_command.task_test(args)
assert f"Marking task as SUCCESS. dag_id={self.dag_id},
task_id={task_id}" in caplog.text
+ @pytest.mark.enable_redact
@pytest.mark.filterwarnings("ignore::airflow.utils.context.AirflowContextDeprecationWarning")
def test_test_filters_secrets(self, capsys):
"""Test ``airflow test`` does not print secrets to stdout.
diff --git a/tests/conftest.py b/tests/conftest.py
index b3efb02fa9..bef998f919 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -402,6 +402,7 @@ def pytest_configure(config: pytest.Config) -> None:
"markers",
"external_python_operator: external python operator tests are 'long',
we should run them separately",
)
+ config.addinivalue_line("markers", "enable_redact: do not mock redact
secret masker")
os.environ["_AIRFLOW__SKIP_DATABASE_EXECUTOR_COMPATIBILITY_CHECK"] = "1"
configure_warning_output(config)
@@ -1234,6 +1235,25 @@ def cleanup_providers_manager():
ProvidersManager()._cleanup()
[email protected](autouse=True)
+def _disable_redact(request: pytest.FixtureRequest, mocker):
+ """Disable redacted text in tests, except specific."""
+ from airflow import settings
+
+ if next(request.node.iter_markers("enable_redact"), None):
+ with pytest.MonkeyPatch.context() as mp_ctx:
+ mp_ctx.setattr(settings, "MASK_SECRETS_IN_LOGS", True)
+ yield
+ return
+
+ mocked_redact =
mocker.patch("airflow.utils.log.secrets_masker.SecretsMasker.redact")
+ mocked_redact.side_effect = lambda item, name=None, max_depth=None: item
+ with pytest.MonkeyPatch.context() as mp_ctx:
+ mp_ctx.setattr(settings, "MASK_SECRETS_IN_LOGS", False)
+ yield
+ return
+
+
# The code below is a modified version of capture-warning code from
# https://github.com/athinkingape/pytest-capture-warnings
diff --git a/tests/models/test_renderedtifields.py
b/tests/models/test_renderedtifields.py
index 0d0ca3be07..3e631a0ba9 100644
--- a/tests/models/test_renderedtifields.py
+++ b/tests/models/test_renderedtifields.py
@@ -169,6 +169,7 @@ class TestRenderedTaskInstanceFields:
# Fetching them will return None
assert RTIF.get_templated_fields(ti=ti2) is None
+ @pytest.mark.enable_redact
def test_secrets_are_masked_when_large_string(self, dag_maker):
"""
Test that secrets are masked when the templated field is a large string
diff --git a/tests/providers/openlineage/plugins/test_utils.py
b/tests/providers/openlineage/plugins/test_utils.py
index 4a9b681733..6062e87b8d 100644
--- a/tests/providers/openlineage/plugins/test_utils.py
+++ b/tests/providers/openlineage/plugins/test_utils.py
@@ -134,6 +134,7 @@ def test_is_name_redactable():
assert _is_name_redactable("transparent", Mixined())
[email protected]_redact
def test_redact_with_exclusions(monkeypatch):
redactor = OpenLineageRedactor.from_masker(_secrets_masker())
diff --git a/tests/utils/log/test_secrets_masker.py
b/tests/utils/log/test_secrets_masker.py
index 03a4736a3c..84170f962e 100644
--- a/tests/utils/log/test_secrets_masker.py
+++ b/tests/utils/log/test_secrets_masker.py
@@ -29,7 +29,6 @@ from unittest.mock import patch
import pytest
-from airflow import settings
from airflow.models import Connection
from airflow.utils.log.secrets_masker import (
RedactedIO,
@@ -41,8 +40,7 @@ from airflow.utils.log.secrets_masker import (
from airflow.utils.state import DagRunState, JobState, State, TaskInstanceState
from tests.test_utils.config import conf_vars
-settings.MASK_SECRETS_IN_LOGS = True
-
+pytestmark = pytest.mark.enable_redact
p = "password"
diff --git a/tests/www/views/test_views_connection.py
b/tests/www/views/test_views_connection.py
index 1c8ccc4259..a209cdfc2b 100644
--- a/tests/www/views/test_views_connection.py
+++ b/tests/www/views/test_views_connection.py
@@ -100,6 +100,7 @@ def test_all_fields_with_blanks(admin_client, session):
assert "airflow" == conn.schema
[email protected]_redact
def test_action_logging_connection_masked_secrets(session, admin_client):
admin_client.post("/connection/add", data=conn_with_extra(),
follow_redirects=True)
_check_last_log_masked_connection(session, dag_id=None,
event="connection.create", execution_date=None)
diff --git a/tests/www/views/test_views_decorators.py
b/tests/www/views/test_views_decorators.py
index 508e7ef32d..f10b3d6684 100644
--- a/tests/www/views/test_views_decorators.py
+++ b/tests/www/views/test_views_decorators.py
@@ -146,6 +146,7 @@ def test_action_logging_variables_post(session,
admin_client):
_check_last_log(session, dag_id=None, event="variable.create",
execution_date=None)
[email protected]_redact
def test_action_logging_variables_masked_secrets(session, admin_client):
form = dict(key="x_secret", val="randomval")
admin_client.post("/variable/add", data=form)
diff --git a/tests/www/views/test_views_rendered.py
b/tests/www/views/test_views_rendered.py
index 9ea7b7fb5c..842f101013 100644
--- a/tests/www/views/test_views_rendered.py
+++ b/tests/www/views/test_views_rendered.py
@@ -229,6 +229,7 @@ def
test_user_defined_filter_and_macros_raise_error(admin_client, create_dag_run
assert "originalerror: no filter named 'hello'" in
resp_html.lower()
[email protected]_redact
@pytest.mark.usefixtures("patch_app")
def test_rendered_template_secret(admin_client, create_dag_run, task_secret):
"""Test that the Rendered View masks values retrieved from secret
variables."""
@@ -261,6 +262,7 @@ else:
initial_db_init()
[email protected]_redact
@pytest.mark.parametrize(
"env, expected",
[
