This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 150ae660a1e Warn when connections/variables list output may be
incomplete (#72484)
150ae660a1e is described below
commit 150ae660a1e2e5bfa4bfef883d2f78c63c416d78
Author: Dheeren Mohta <[email protected]>
AuthorDate: Wed Sep 23 04:39:08 2026 +0530
Warn when connections/variables list output may be incomplete (#72484)
* Warn when connections/variables list output may be incomplete
`airflow connections list` and `airflow variables list` only enumerate
rows in the metadata database. Connections and variables can also be
defined via `AIRFLOW_CONN_*`/`AIRFLOW_VAR_*` environment variables or a
configured secrets backend, and those sources are checked ahead of the
database when Airflow actually resolves a connection or variable at
runtime -- so an environment variable or secrets-backend entry can
silently override a database row with the same ID without ever
appearing in the list output. Users had no indication the listing was
incomplete, making it easy to assume the database is the only source
of truth.
This adds a stderr warning to both list commands (so it doesn't
interfere with `--output json`/`yaml` piping) whenever an
`AIRFLOW_CONN_*`/`AIRFLOW_VAR_*` environment variable or a secrets
backend is present, naming the source(s) that may hold additional or
overriding entries.
This covers the CLI surface only. Extending the same warning to the
Web UI would require adding fields to the REST API's connection/variable
response models, which are generated from an OpenAPI spec and would
require regenerating the TypeScript client -- a much larger, separate
change better suited to its own PR.
Related: #10867
* Address review feedback on the hidden-entries warning
Workers can override the general secrets backend with their own
[workers] secrets_backend setting, which the Task SDK prioritizes
during secret resolution; the CLI warning missed that source. Checking
both keys keeps the warning accurate for worker-backed deployments.
The import moves to module scope since there is no cycle with
airflow.configuration, and the wording drops knowledge of which
specific backend is configured.
* Address maintainer follow-up review on the hidden-entries warning
The secrets-backend (section, key) pairs now live as a single constant
in the shared parser, read by both _get_custom_secret_backend() and
this warning, so a new backend source only needs to be added once.
Also fixes a test comment that named the wrong fixture as the source
of ambient AIRFLOW_CONN_* env vars, and pins [secrets] backend /
[workers] secrets_backend to empty in the does-not-warn-by-default
tests so they don't depend on the ambient shell environment.
* Fix CodeQL false positive from a secrets-named config-key constant
CodeQL's clear-text-logging query treats any value sourced from an
identifier containing "secret" as sensitive data. The (section, key)
lookup table added in the previous commit was named
SECRETS_BACKEND_CONFIG_KEYS, so section/key strings pulled from it
tripped that heuristic wherever they later reach a log statement
(_get_custom_secret_backend()'s parse-failure warnings, and the
generic config lookup's not-found warning) — even though the values
themselves are just config coordinates like "secrets"/"backend", never
the secret they point at. Renaming the constant to
CUSTOM_BACKEND_CONFIG_KEYS removes the name-based trigger without
changing any behavior.
---
.../src/airflow/cli/commands/connection_command.py | 5 ++
.../src/airflow/cli/commands/variable_command.py | 11 +++-
airflow-core/src/airflow/cli/utils.py | 40 ++++++++++++++
.../unit/cli/commands/test_connection_command.py | 32 +++++++++++-
.../unit/cli/commands/test_variable_command.py | 27 +++++++++-
airflow-core/tests/unit/cli/test_utils.py | 61 +++++++++++++++++++++-
.../src/airflow_shared/configuration/parser.py | 16 ++++--
7 files changed, 184 insertions(+), 8 deletions(-)
diff --git a/airflow-core/src/airflow/cli/commands/connection_command.py
b/airflow-core/src/airflow/cli/commands/connection_command.py
index 26bf7c127c5..99dd3699df9 100644
--- a/airflow-core/src/airflow/cli/commands/connection_command.py
+++ b/airflow-core/src/airflow/cli/commands/connection_command.py
@@ -34,6 +34,7 @@ from airflow.cli.simple_table import AirflowConsole
from airflow.cli.utils import (
SENSITIVE_PLACEHOLDER,
deprecated_for_airflowctl,
+ get_hidden_entries_warning,
is_stdout,
print_export_output,
)
@@ -41,6 +42,7 @@ from airflow.configuration import conf
from airflow.exceptions import AirflowNotFoundException
from airflow.models import Connection
from airflow.providers_manager import ProvidersManager
+from airflow.secrets.environment_variables import CONN_ENV_PREFIX
from airflow.secrets.local_filesystem import load_connections_dict
from airflow.utils import cli as cli_utils, helpers, yaml
from airflow.utils.cli import suppress_logs_and_warning
@@ -169,6 +171,9 @@ def connections_list(args):
else:
mapper = ConnectionDisplayMapper.full_details
+ if warning := get_hidden_entries_warning("connections", CONN_ENV_PREFIX):
+ AirflowConsole(stderr=True).print(f"[bold yellow]Warning:[/bold
yellow] {warning}\n")
+
with create_session() as session:
query = select(Connection)
conns = session.scalars(query).all()
diff --git a/airflow-core/src/airflow/cli/commands/variable_command.py
b/airflow-core/src/airflow/cli/commands/variable_command.py
index 8cdf5490d3a..c397f3a4d6f 100644
--- a/airflow-core/src/airflow/cli/commands/variable_command.py
+++ b/airflow-core/src/airflow/cli/commands/variable_command.py
@@ -26,13 +26,19 @@ from typing import TYPE_CHECKING
from sqlalchemy import select
from airflow.cli.simple_table import AirflowConsole
-from airflow.cli.utils import SENSITIVE_PLACEHOLDER,
deprecated_for_airflowctl, print_export_output
+from airflow.cli.utils import (
+ SENSITIVE_PLACEHOLDER,
+ deprecated_for_airflowctl,
+ get_hidden_entries_warning,
+ print_export_output,
+)
from airflow.exceptions import (
AirflowFileParseException,
AirflowUnsupportedFileTypeException,
VariableNotUnique,
)
from airflow.models import Variable
+from airflow.secrets.environment_variables import VAR_ENV_PREFIX
from airflow.secrets.local_filesystem import load_variables
from airflow.utils import cli as cli_utils
from airflow.utils.cli import suppress_logs_and_warning
@@ -83,6 +89,9 @@ def variables_list(args):
def _mapper(var):
return VariableDisplayMapper.with_values(var, hide_sensitive)
+ if warning := get_hidden_entries_warning("variables", VAR_ENV_PREFIX):
+ AirflowConsole(stderr=True).print(f"[bold yellow]Warning:[/bold
yellow] {warning}\n")
+
with create_session() as session:
if show_values:
variables = session.scalars(select(Variable)).all()
diff --git a/airflow-core/src/airflow/cli/utils.py
b/airflow-core/src/airflow/cli/utils.py
index aef66a88046..098ff9d498a 100644
--- a/airflow-core/src/airflow/cli/utils.py
+++ b/airflow-core/src/airflow/cli/utils.py
@@ -18,10 +18,14 @@
from __future__ import annotations
import logging
+import os
import sys
from collections.abc import Callable
from typing import TYPE_CHECKING, TypeVar
+from airflow._shared.configuration.parser import CUSTOM_BACKEND_CONFIG_KEYS
+from airflow.configuration import conf
+
# Placeholder for masking sensitive values in CLI output
SENSITIVE_PLACEHOLDER = "***"
@@ -100,6 +104,42 @@ def print_export_output(command_type: str, exported_items:
Collection, file: Tex
print(f"{len(exported_items)} {command_type} successfully exported to
{file.name}.")
+def get_hidden_entries_warning(entity_name: str, env_prefix: str) -> str |
None:
+ """
+ Return a warning when the database listing may be incomplete.
+
+ :param entity_name: Human-readable plural noun to use in the message, e.g.
``"connections"``.
+ :param env_prefix: Environment variable prefix used for this entity, e.g.
``AIRFLOW_CONN_``.
+ :return: A warning message, or ``None`` if neither hiding source appears
to be in use.
+ """
+ # Connections and variables may also come from environment variables or a
+ # custom secrets backend. These sources can override database entries but
+ # are not included by commands that enumerate database rows.
+ has_env_vars = any(key.startswith(env_prefix) for key in os.environ)
+ # Only check whether custom backends are *configured*, without
instantiating them (which could
+ # have side effects, e.g. opening a network connection to a Vault/AWS/GCP
secrets service).
+ # Reads the same (section, key) pairs _get_custom_secret_backend() uses,
so a new backend
+ # source only needs to be added in one place.
+ has_secrets_backend = any(
+ conf.get(section, key, fallback=None) for section, key in
CUSTOM_BACKEND_CONFIG_KEYS.values()
+ )
+
+ if not has_env_vars and not has_secrets_backend:
+ return None
+
+ sources = []
+ if has_env_vars:
+ sources.append(f"`{env_prefix}*` environment variables")
+ if has_secrets_backend:
+ sources.append("a configured secrets backend")
+
+ return (
+ f"This list only includes {entity_name} stored in the metadata
database. "
+ f"{' and '.join(sources)} may also define {entity_name} -- including
ones that override a "
+ "database entry with the same ID -- that will not appear here."
+ )
+
+
def fetch_dag_run_from_run_id_or_logical_date_string(
*,
dag_id: str,
diff --git a/airflow-core/tests/unit/cli/commands/test_connection_command.py
b/airflow-core/tests/unit/cli/commands/test_connection_command.py
index fab54ce782f..a923501caaf 100644
--- a/airflow-core/tests/unit/cli/commands/test_connection_command.py
+++ b/airflow-core/tests/unit/cli/commands/test_connection_command.py
@@ -21,7 +21,7 @@ import os
import re
import shlex
import warnings
-from contextlib import redirect_stdout
+from contextlib import redirect_stderr, redirect_stdout
from io import StringIO
import pytest
@@ -35,6 +35,7 @@ from airflow.models import Connection
from airflow.utils.db import merge_conn
from airflow.utils.session import create_session
+from tests_common.test_utils.config import conf_vars
from tests_common.test_utils.db import clear_db_connections
from tests_common.test_utils.markers import
skip_if_force_lowest_dependencies_marker
@@ -147,6 +148,35 @@ class TestCliListConnections:
with pytest.raises(SystemExit, match="--hide-sensitive can only be
used with --show-values"):
connection_command.connections_list(args)
+ def test_cli_connections_list_warns_about_env_var_connections(self,
monkeypatch):
+ """An `AIRFLOW_CONN_*` environment variable should trigger a stderr
warning."""
+ # The module-level database cleanup fixture may seed default
`AIRFLOW_CONN_*`
+ # variables, so remove that ambient state before testing this specific
entry.
+ for key in list(os.environ):
+ if key.startswith("AIRFLOW_CONN_"):
+ monkeypatch.delenv(key, raising=False)
+ monkeypatch.setenv("AIRFLOW_CONN_MY_HIDDEN_CONN",
"postgresql://u:p@host/db")
+ args = self.parser.parse_args(["connections", "list", "--output",
"json"])
+ with redirect_stderr(StringIO()) as stderr_io:
+ connection_command.connections_list(args)
+ stderr = stderr_io.getvalue()
+ assert "AIRFLOW_CONN_" in stderr
+ assert "metadata database" in stderr
+
+ def test_cli_connections_list_does_not_warn_by_default(self, monkeypatch):
+ """With no env-var connections or secrets backend configured, no
warning is printed."""
+ for key in list(os.environ):
+ if key.startswith("AIRFLOW_CONN_"):
+ monkeypatch.delenv(key, raising=False)
+ args = self.parser.parse_args(["connections", "list", "--output",
"json"])
+ with (
+ conf_vars({("secrets", "backend"): "", ("workers",
"secrets_backend"): ""}),
+ redirect_stderr(StringIO()) as stderr_io,
+ ):
+ connection_command.connections_list(args)
+ stderr = stderr_io.getvalue()
+ assert stderr == ""
+
class TestUriMasking:
"""Test URI credential masking functionality."""
diff --git a/airflow-core/tests/unit/cli/commands/test_variable_command.py
b/airflow-core/tests/unit/cli/commands/test_variable_command.py
index ac8eb11a3ae..76f39a2eeba 100644
--- a/airflow-core/tests/unit/cli/commands/test_variable_command.py
+++ b/airflow-core/tests/unit/cli/commands/test_variable_command.py
@@ -19,7 +19,7 @@ from __future__ import annotations
import json
import os
-from contextlib import redirect_stdout
+from contextlib import redirect_stderr, redirect_stdout
from io import StringIO
import pytest
@@ -32,6 +32,7 @@ from airflow.cli.commands import variable_command
from airflow.models import Variable
from airflow.utils.session import create_session
+from tests_common.test_utils.config import conf_vars
from tests_common.test_utils.db import clear_db_variables
pytestmark = pytest.mark.db_test
@@ -324,6 +325,30 @@ class TestCliVariables:
if item["key"] in ["empty_var", "none_var", "normal_var"]:
assert item["val"] == "***"
+ def test_variables_list_warns_about_env_var_variables(self, monkeypatch):
+ """An `AIRFLOW_VAR_*` environment variable should trigger a stderr
warning."""
+ monkeypatch.setenv("AIRFLOW_VAR_MY_HIDDEN_VAR", "hidden_value")
+ args = self.parser.parse_args(["variables", "list", "--output",
"json"])
+ with redirect_stderr(StringIO()) as stderr_io:
+ variable_command.variables_list(args)
+ stderr = stderr_io.getvalue()
+ assert "AIRFLOW_VAR_" in stderr
+ assert "metadata database" in stderr
+
+ def test_variables_list_does_not_warn_by_default(self, monkeypatch):
+ """With no env-var variables or secrets backend configured, no warning
is printed."""
+ for key in list(os.environ):
+ if key.startswith("AIRFLOW_VAR_"):
+ monkeypatch.delenv(key, raising=False)
+ args = self.parser.parse_args(["variables", "list", "--output",
"json"])
+ with (
+ conf_vars({("secrets", "backend"): "", ("workers",
"secrets_backend"): ""}),
+ redirect_stderr(StringIO()) as stderr_io,
+ ):
+ variable_command.variables_list(args)
+ stderr = stderr_io.getvalue()
+ assert stderr == ""
+
def test_variables_delete(self):
"""Test variable_delete command"""
variable_command.variables_set(self.parser.parse_args(["variables",
"set", "foo", "bar"]))
diff --git a/airflow-core/tests/unit/cli/test_utils.py
b/airflow-core/tests/unit/cli/test_utils.py
index bd1556dd310..ea3e987676f 100644
--- a/airflow-core/tests/unit/cli/test_utils.py
+++ b/airflow-core/tests/unit/cli/test_utils.py
@@ -18,12 +18,19 @@
from __future__ import annotations
import logging
+import os
import sys
import warnings
import pytest
-from airflow.cli.utils import deprecated_for_airflowctl,
redirect_stdout_log_handlers_to_stderr
+from airflow.cli.utils import (
+ deprecated_for_airflowctl,
+ get_hidden_entries_warning,
+ redirect_stdout_log_handlers_to_stderr,
+)
+
+from tests_common.test_utils.config import conf_vars
class TestDeprecatedForAirflowctl:
@@ -97,3 +104,55 @@ class TestRedirectStdoutLogHandlersToStderr:
assert handler.stream is expected_stream(original_stream)
finally:
handler.close()
+
+
+class TestGetHiddenEntriesWarning:
+ """Tests for the CLI warning about connections/variables hidden from
``list`` output."""
+
+ @pytest.fixture(autouse=True)
+ def _clear_ambient_env_prefixes(self, monkeypatch):
+ """Strip any ``AIRFLOW_CONN_*``/``AIRFLOW_VAR_*`` vars already present
so tests are isolated."""
+ for key in list(os.environ):
+ if key.startswith(("AIRFLOW_CONN_", "AIRFLOW_VAR_")):
+ monkeypatch.delenv(key, raising=False)
+
+ def test_returns_none_when_nothing_is_hidden(self):
+ with conf_vars({("secrets", "backend"): "", ("workers",
"secrets_backend"): ""}):
+ assert get_hidden_entries_warning("connections", "AIRFLOW_CONN_")
is None
+
+ def test_warns_about_env_var_defined_entries(self, monkeypatch):
+ monkeypatch.setenv("AIRFLOW_CONN_MY_DB", "postgresql://u:p@host/db")
+ with conf_vars({("secrets", "backend"): "", ("workers",
"secrets_backend"): ""}):
+ warning = get_hidden_entries_warning("connections",
"AIRFLOW_CONN_")
+
+ assert warning is not None
+ assert "AIRFLOW_CONN_" in warning
+ assert "secrets backend" not in warning
+
+ def test_warns_about_configured_secrets_backend(self):
+ with conf_vars({("secrets", "backend"):
"airflow.secrets.local_filesystem.LocalFilesystemBackend"}):
+ warning = get_hidden_entries_warning("variables", "AIRFLOW_VAR_")
+
+ assert warning is not None
+ assert "secrets backend" in warning
+ assert "AIRFLOW_VAR_" not in warning
+
+ def test_warns_about_worker_secrets_backend(self):
+ with conf_vars(
+ {("workers", "secrets_backend"):
"airflow.secrets.local_filesystem.LocalFilesystemBackend"}
+ ):
+ warning = get_hidden_entries_warning("variables", "AIRFLOW_VAR_")
+
+ assert warning is not None
+ assert "secrets backend" in warning
+ assert "AIRFLOW_VAR_" not in warning
+
+ def test_warns_about_both_sources_when_both_are_present(self, monkeypatch):
+ monkeypatch.setenv("AIRFLOW_VAR_MY_KEY", "value")
+ with conf_vars({("secrets", "backend"):
"airflow.secrets.local_filesystem.LocalFilesystemBackend"}):
+ warning = get_hidden_entries_warning("variables", "AIRFLOW_VAR_")
+
+ assert warning is not None
+ assert "AIRFLOW_VAR_" in warning
+ assert "secrets backend" in warning
+ assert " and " in warning
diff --git a/shared/configuration/src/airflow_shared/configuration/parser.py
b/shared/configuration/src/airflow_shared/configuration/parser.py
index 7222c0212cc..70282cc0e2c 100644
--- a/shared/configuration/src/airflow_shared/configuration/parser.py
+++ b/shared/configuration/src/airflow_shared/configuration/parser.py
@@ -79,6 +79,15 @@ ConfigSourcesType = dict[str, ConfigSectionSourcesType]
ENV_VAR_PREFIX = "AIRFLOW__"
# Separates the team name from the base section name in a team scoped config
file section.
TEAM_SECTION_SEPARATOR = "="
+# (section, key) pairs that may hold a custom secrets backend class.
_get_custom_secret_backend()
+# reads this to pick the pair for the current mode; callers that only need to
know whether *some*
+# backend is configured (e.g. a CLI warning), without instantiating one, read
all of the values.
+# Named without "secret" so static analysis (e.g. CodeQL's clear-text-logging
query) doesn't treat
+# the (non-secret) section/key strings sourced from here as sensitive data.
+CUSTOM_BACKEND_CONFIG_KEYS = {
+ "general": ("secrets", "backend"),
+ "worker": ("workers", "secrets_backend"),
+}
def team_section_name(team_name: str, section: str) -> str:
@@ -723,8 +732,7 @@ class AirflowConfigParser(ConfigParser):
Conditionally selects the section, key and kwargs key based on whether
it is called from worker or not.
"""
- section = "workers" if worker_mode else "secrets"
- key = "secrets_backend" if worker_mode else "backend"
+ section, key = CUSTOM_BACKEND_CONFIG_KEYS["worker" if worker_mode else
"general"]
kwargs_key = "secrets_backend_kwargs" if worker_mode else
"backend_kwargs"
secrets_backend_cls = self.getimport(section=section, key=key)
@@ -732,12 +740,12 @@ class AirflowConfigParser(ConfigParser):
if not secrets_backend_cls:
if worker_mode:
# if we find no secrets backend for worker, return that of
secrets backend
- secrets_backend_cls = self.getimport(section="secrets",
key="backend")
+ section, key = CUSTOM_BACKEND_CONFIG_KEYS["general"]
+ secrets_backend_cls = self.getimport(section=section, key=key)
if not secrets_backend_cls:
return None
# When falling back to secrets backend, use its kwargs
kwargs_key = "backend_kwargs"
- section = "secrets"
else:
return None
