This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c6a4acf8cde Fix multi-line secrets leaking in logs formatted with
pprint (#75651)
c6a4acf8cde is described below
commit c6a4acf8cde88ee11a805435c1de3f3fb707e909
Author: juan-pablo-guereca <[email protected]>
AuthorDate: Thu Nov 12 20:21:39 2026 +0100
Fix multi-line secrets leaking in logs formatted with pprint (#75651)
* Mask multi-line secrets when logged through pprint-style formatting
Secrets such as private keys were left readable in task logs when
a Kubernetes pod spec was formatted into an exception message,
because pprint splits a multi-line string into separate literals and
the masker only looked for the secret as one contiguous string.
* Make the multi-line secret masking test independent of pprint
The regression test built its input with pprint.pformat, so a change in
how pprint wraps strings could make the test pass without exercising
the masker. Spell out the formatted text and the expected result in the
test instead, and use raw strings for the expected regex patterns.
* Re-trigger CI after a runner failure
One MySQL test job on the previous commit failed with "The hosted
runner lost communication with the server". It passed on the commit
before and the other database jobs passed, but contributors cannot
re-run individual jobs, so push an empty commit to start a new run.
* Mask secrets that end with a line break
A value read from a file usually ends with a newline, and pprint prints
it with the newline escaped, so the registered secret never matched and
stayed readable in logs. Register the stripped value of any secret that
contains a line break, not only of secrets with several lines.
* Document per-line masking of multi-line values
Multi-line secrets are now also masked line by line, which can replace
ordinary text when a pretty-printed connection extra is registered.
Describe both effects so readers who find *** in place of plain text
know where it comes from.
---
.../security/secrets/mask-sensitive-values.rst | 11 +++++
.../secrets_masker/secrets_masker.py | 6 +++
.../tests/secrets_masker/test_secrets_masker.py | 51 ++++++++++++++++++++++
3 files changed, 68 insertions(+)
diff --git a/airflow-core/docs/security/secrets/mask-sensitive-values.rst
b/airflow-core/docs/security/secrets/mask-sensitive-values.rst
index 780aceb70fb..ae189dc5dfc 100644
--- a/airflow-core/docs/security/secrets/mask-sensitive-values.rst
+++ b/airflow-core/docs/security/secrets/mask-sensitive-values.rst
@@ -29,6 +29,17 @@ It does this by looking for the specific *value* appearing
anywhere in your outp
have a connection with a password of ``a``, then every instance of the letter
a in your logs will be replaced
with ``***``.
+A value that spans several lines, such as a private key, is also masked line
by line: each eligible non-blank line,
+with surrounding whitespace removed, is masked in addition to the whole value.
This also masks matching lines when
+a formatter such as ``pprint`` splits a multi-line string into separate string
literals, as happens when a
+Kubernetes pod spec is printed. Lines shorter than
:ref:`config:logging__min_length_masked_secret` are not masked
+on their own.
+
+This also applies to a Connection's extra field that is stored as
pretty-printed, multi-line JSON: each eligible
+line of it, such as ``"timeout": 30,``, is replaced with ``***`` in your logs,
including in unrelated text.
+Compact, single-line JSON does not have this effect, but saving the connection
in the UI formats extra with
+indentation again.
+
To disable masking you can set
:ref:`config:core__hide_sensitive_var_conn_fields` to false.
The automatic masking is triggered by Connection or Variable access. This
means that if you pass a sensitive
diff --git
a/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
b/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
index d34d9b0a263..63d2827ca04 100644
--- a/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
+++ b/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
@@ -592,6 +592,12 @@ class SecretsMasker(logging.Filter):
"""Yield the secret along with any adaptations to the secret that
should be masked."""
yield secret
+ # Formatters such as ``pprint`` (used by ``str()`` of Kubernetes
models) escape newlines and split a
+ # multi-line string across several literals, so the secret never
appears contiguously in the output.
+ lines = secret.splitlines()
+ if lines != [secret]:
+ yield from map(str.strip, lines)
+
if self.secret_mask_adapter:
# This can return an iterable of secrets to mask OR a single
secret as a string
secret_or_secrets = self.secret_mask_adapter(secret)
diff --git a/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
b/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
index 1d692e53faa..fac109a4231 100644
--- a/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
+++ b/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
@@ -1266,6 +1266,57 @@ class TestEdgeCases:
assert redacted.startswith("Contains ")
assert " and " in redacted
+ def test_multiline_secret_masked_when_split_into_adjacent_literals(self):
+ secret = "-----BEGIN KEY-----\nabcDEF123\nxyz789\n-----END KEY-----"
+ secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
+ secrets_masker.add_mask(secret)
+
+ formatted = textwrap.dedent(
+ r"""
+ {'name': 'MULTI',
+ 'value': '-----BEGIN KEY-----\n'
+ 'abcDEF123\n'
+ 'xyz789\n'
+ '-----END KEY-----'}
+ """
+ )
+ expected = textwrap.dedent(
+ r"""
+ {'name': 'MULTI',
+ 'value': '***\n'
+ '***\n'
+ '***\n'
+ '***'}
+ """
+ )
+
+ assert secrets_masker.redact(formatted) == expected
+
+ @pytest.mark.parametrize("newline", ["\n", "\r\n"])
+ def test_multiline_secret_registers_stripped_lines(self, newline):
+ secret = newline.join([" first-line-1 ", "", "second-line-2"])
+ secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
+ secrets_masker.add_mask(secret)
+
+ assert {r"first\-line\-1", r"second\-line\-2"} <=
secrets_masker.patterns
+ assert secrets_masker.redact(f"x {secret} y") == "x *** y"
+
+ single_line_masker = SecretsMasker()
+ configure_secrets_masker_for_test(single_line_masker)
+ single_line_masker.add_mask(" error ")
+ assert single_line_masker.patterns == {r"\ \ error\ \ "}
+
+ @pytest.mark.parametrize(("newline", "escaped"), [("\n", r"\n"), ("\r\n",
r"\r\n")])
+ def test_secret_with_trailing_newline_registers_stripped_value(self,
newline, escaped):
+ secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
+ secrets_masker.add_mask(f"token-value{newline}")
+
+ assert r"token\-value" in secrets_masker.patterns
+ assert secrets_masker.redact(f"{{'v': 'token-value{escaped}'}}") ==
f"{{'v': '***{escaped}'}}"
+
class TestDirectMethodCalls:
def test_redact_all_directly(self):
