This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4d39a855b1c Walk nested lists/tuples/sets in secrets masker key-name
redaction (#68422)
4d39a855b1c is described below
commit 4d39a855b1c6c3072e53b49465dc66f5a38e095a
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Jun 25 04:22:23 2026 -0400
Walk nested lists/tuples/sets in secrets masker key-name redaction (#68422)
The secrets masker walks dicts unconditionally for key-name-based
redaction, so a sensitive key nested in dicts is masked at any depth.
Lists, tuples, and sets were only walked below the recursion-depth
cutoff, so a value carrying a sensitive key wrapped in an iterable
beyond the cutoff was returned without key-name redaction.
Walk nested lists/tuples/sets unconditionally too, mirroring the dict
handling, keeping the depth cutoff only for pattern-based string
masking. Add regression tests for sensitive keys wrapped in lists and
tuples past MAX_RECURSION_DEPTH.
Generated-by: Claude Opus 4.8 (1M context) following the guidelines at
https://github.com/apache/airflow/blob/main/contributing-docs/05_pull_requests.rst#gen-ai-assisted-contributions
---
.../secrets_masker/secrets_masker.py | 40 +++++++++++++---------
.../tests/secrets_masker/test_secrets_masker.py | 14 ++++++++
2 files changed, 37 insertions(+), 17 deletions(-)
diff --git
a/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
b/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
index 3b625b7af77..9157a91a7fb 100644
--- a/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
+++ b/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
@@ -369,8 +369,29 @@ class SecretsMasker(logging.Filter):
for dict_key, subval in item.items()
}
return to_return
- # Avoid spending too much effort on pattern-based masking of
- # deeply nested non-dict structures.
+ # Always walk lists/tuples/sets too, mirroring the unconditional
dict
+ # walk above, so a sensitive key wrapped in an iterable is still
+ # caught at any nesting depth. Self-referential iterables hit
Python's
+ # own recursion limit and are caught by the except clause below,
which
+ # fails closed.
+ if isinstance(item, (tuple, set)):
+ # Turn set in to tuple!
+ return tuple(
+ self._redact(
+ subval, name=None, depth=(depth + 1),
max_depth=max_depth, replacement=replacement
+ )
+ for subval in item
+ )
+ if isinstance(item, list):
+ return [
+ self._redact(
+ subval, name=None, depth=(depth + 1),
max_depth=max_depth, replacement=replacement
+ )
+ for subval in item
+ ]
+ # The depth cutoff only bounds the work of pattern-based string
+ # masking below — key-name redaction (dicts and iterables above) is
+ # unbounded so sensitive keys fail closed at any depth.
if depth > max_depth:
return item
if isinstance(item, Enum):
@@ -393,21 +414,6 @@ class SecretsMasker(logging.Filter):
# the structure.
return self.replacer.sub(replacement, str(item))
return item
- if isinstance(item, (tuple, set)):
- # Turn set in to tuple!
- return tuple(
- self._redact(
- subval, name=None, depth=(depth + 1),
max_depth=max_depth, replacement=replacement
- )
- for subval in item
- )
- if isinstance(item, list):
- return [
- self._redact(
- subval, name=None, depth=(depth + 1),
max_depth=max_depth, replacement=replacement
- )
- for subval in item
- ]
return item
# I think this should never happen, but it does not hurt to leave it
just in case
# Well. It happened (see
https://github.com/apache/airflow/issues/19816#issuecomment-983311373)
diff --git a/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
b/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
index c6990602284..b3b40475a20 100644
--- a/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
+++ b/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
@@ -718,6 +718,20 @@ class TestSecretsMasker:
{"a": {"b": {"c": {"d": {"e": {"api_key": "leaked"}}}}}},
{"a": {"b": {"c": {"d": {"e": {"api_key": "***"}}}}}},
),
+ # A sensitive key wrapped in a list beyond MAX_RECURSION_DEPTH is
+ # still redacted: the list is walked unconditionally, mirroring the
+ # unbounded dict walk above. Here the list sits at depth 6 (one
past
+ # the cutoff), reached through the unbounded dict chain a..f.
+ (
+ {"a": {"b": {"c": {"d": {"e": {"f": [{"password":
"leaked"}]}}}}}},
+ {"a": {"b": {"c": {"d": {"e": {"f": [{"password":
"***"}]}}}}}},
+ ),
+ # Same for a tuple-wrapped sensitive key past MAX_RECURSION_DEPTH
+ # (a set cannot hold a dict, so only list/tuple are exercised
here).
+ (
+ {"a": {"b": {"c": {"d": {"e": {"f": ({"token":
"leaked"},)}}}}}},
+ {"a": {"b": {"c": {"d": {"e": {"f": ({"token": "***"},)}}}}}},
+ ),
],
)
def test_redact_sensitive_key_past_max_depth(self, val, expected):
