This is an automated email from the ASF dual-hosted git repository.
o-nikolas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4744db9eca9 Warn when AwsDuckDBHook ignores the AWS connection's
credentials (#76639)
4744db9eca9 is described below
commit 4744db9eca920580aa3b1f3b335c3bdf50e2dc2d
Author: Sean Ghaeli <[email protected]>
AuthorDate: Mon Nov 9 14:12:15 2026 -0700
Warn when AwsDuckDBHook ignores the AWS connection's credentials (#76639)
With the default credential_chain strategy, keys, a role or a profile in
the AWS connection are never used: DuckDB resolves credentials on its own.
That is easy to miss, since every other AWS hook uses them.
When credential_strategy is not set and the AWS connection has
credentials, log a warning that points at config (to use them) and
credential_chain (to silence it). The default and the secret are
unchanged.
---
providers/amazon/docs/operators/duckdb.rst | 4 ++
.../airflow/providers/amazon/aws/hooks/duckdb.py | 28 ++++++++++++++
.../tests/unit/amazon/aws/hooks/test_duckdb.py | 44 +++++++++++++++++++++-
3 files changed, 75 insertions(+), 1 deletion(-)
diff --git a/providers/amazon/docs/operators/duckdb.rst
b/providers/amazon/docs/operators/duckdb.rst
index 2afea6a11a1..d50ff94f6e9 100644
--- a/providers/amazon/docs/operators/duckdb.rst
+++ b/providers/amazon/docs/operators/duckdb.rst
@@ -168,6 +168,10 @@ DuckDB how to authenticate, and the
``credential_strategy`` parameter selects ho
hook_params={"aws_conn_id": "aws_static_keys", "credential_strategy":
"config"},
)
+If ``credential_strategy`` is not set and the AWS connection has credentials
(keys, a ``role_arn`` or
+a ``profile_name``), the hook logs a warning, because ``credential_chain``
does not use them. Set it to
+``config`` to use them, or to ``credential_chain`` to silence the warning.
+
Non-default S3 endpoints
========================
diff --git a/providers/amazon/src/airflow/providers/amazon/aws/hooks/duckdb.py
b/providers/amazon/src/airflow/providers/amazon/aws/hooks/duckdb.py
index d9c9e87296f..2c6f02b663b 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/hooks/duckdb.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/hooks/duckdb.py
@@ -73,6 +73,9 @@ class AwsDuckDBHook(DuckDBHook):
running inside DuckDB cannot see, or when talking to an S3-compatible
endpoint with its own
credentials. Prefer ``credential_chain`` where it works.
+ When ``credential_strategy`` is not set and the AWS connection has
credentials (keys, a
+ ``role_arn`` or a ``profile_name``), the hook warns that
``credential_chain`` does not use them.
+
Either strategy is better than none: DuckDB's ``httpfs`` extension has its
own HTTP client which
reads credentials from the standard ``AWS_*`` environment variables but
does **not** implement the
ECS container credential provider, so an ambient task role is invisible to
it and S3 access fails
@@ -145,6 +148,7 @@ class AwsDuckDBHook(DuckDBHook):
self._s3_endpoint_url = s3_endpoint_url
self._secret_name = secret_name
self._aws_identity_looked_up = False
+ self._ignored_credentials_checked = False
@property
def aws_conn_id(self) -> str | None:
@@ -210,6 +214,7 @@ class AwsDuckDBHook(DuckDBHook):
self.log.debug("credential_strategy is 'none'; not creating a
DuckDB S3 secret.")
return
+ self._warn_if_connection_credentials_are_ignored()
self._validate_identifier(self.secret_name, "secret")
parts = ["TYPE s3", *self._provider_clauses(strategy)]
@@ -241,6 +246,29 @@ class AwsDuckDBHook(DuckDBHook):
self._log_aws_identity(strategy, region, endpoint_url)
+ def _warn_if_connection_credentials_are_ignored(self) -> None:
+ """Warn once per hook when ``credential_chain`` leaves the AWS
connection's credentials unused."""
+ if self._ignored_credentials_checked:
+ return
+ self._ignored_credentials_checked = True
+ explicit = self._credential_strategy or
self.connection_extra.get("credential_strategy")
+ if explicit or self.aws_conn_id is None:
+ return
+ # profile_name counts: boto3 resolves it like every other AWS hook,
DuckDB's chain would not.
+ conn_config = self.aws_hook.conn_config
+ if (
+ conn_config.aws_access_key_id
+ or conn_config.aws_session_token
+ or conn_config.role_arn
+ or conn_config.profile_name
+ ):
+ self.log.warning(
+ "The AWS connection %r has credentials, but DuckDB is using
its own credential chain, so "
+ "they are not used. Set credential_strategy to 'config' to use
them, or to "
+ "'credential_chain' to silence this warning.",
+ self.aws_conn_id,
+ )
+
def _log_aws_identity(
self, strategy: CredentialStrategy, region: str | None, endpoint_url:
str | None
) -> None:
diff --git a/providers/amazon/tests/unit/amazon/aws/hooks/test_duckdb.py
b/providers/amazon/tests/unit/amazon/aws/hooks/test_duckdb.py
index c64aa0e1f62..7ffc9a4ba86 100644
--- a/providers/amazon/tests/unit/amazon/aws/hooks/test_duckdb.py
+++ b/providers/amazon/tests/unit/amazon/aws/hooks/test_duckdb.py
@@ -26,6 +26,7 @@ from botocore.exceptions import EndpointConnectionError
from airflow.models.connection import Connection
from airflow.providers.amazon.aws.hooks.duckdb import AwsDuckDBHook
+from airflow.providers.amazon.aws.utils.connection_wrapper import
AwsConnectionWrapper
from airflow.providers.duckdb.hooks.duckdb import DuckDBHook
from airflow.providers.duckdb.operators.duckdb import
DuckDBExecuteQueryOperator
@@ -34,6 +35,7 @@ AWS_BASE_HOOK =
"airflow.providers.amazon.aws.hooks.duckdb.AwsBaseHook"
STS_HOOK = "airflow.providers.amazon.aws.hooks.duckdb.StsHook"
ARN = "arn:aws:sts::123456789012:assumed-role/airflow-worker/session"
S3_VPC_ENDPOINT =
"https://bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com";
+AWS_KEYS = {"login": "AKIA", "password": "s"}
try:
from airflow.sdk.exceptions import AirflowNotFoundException
@@ -47,7 +49,7 @@ def aws_base_hook():
with mock.patch(AWS_BASE_HOOK) as patched:
instance = patched.return_value
instance.region_name = "us-west-2"
- instance.conn_config.endpoint_url = None
+ instance.conn_config = AwsConnectionWrapper(conn=None)
credentials = instance.get_credentials.return_value
credentials.access_key = "AKIAEXAMPLE"
credentials.secret_key = "secret"
@@ -138,6 +140,36 @@ class TestAwsDuckDBHookCredentials:
assert "PROVIDER credential_chain" in statement
assert "KEY_ID" not in statement
+ @pytest.mark.parametrize(
+ ("aws_conn", "hook_kwargs", "warns"),
+ [
+ pytest.param(AWS_KEYS, {}, True, id="keys"),
+ pytest.param({"extra": '{"aws_session_token": "t"}'}, {}, True,
id="session-token"),
+ pytest.param({"extra": '{"role_arn": "arn:aws:iam::1:role/r"}'},
{}, True, id="role"),
+ pytest.param({"extra": '{"profile_name": "p"}'}, {}, True,
id="profile"),
+ pytest.param({"extra": '{"region_name": "eu-west-1"}'}, {}, False,
id="no-credentials"),
+ pytest.param(None, {}, False, id="missing-connection"),
+ pytest.param(AWS_KEYS, {"aws_conn_id": None}, False,
id="no-aws-conn"),
+ pytest.param(AWS_KEYS, {"credential_strategy":
"credential_chain"}, False, id="chain"),
+ pytest.param(AWS_KEYS, {"credential_strategy": "config"}, False,
id="config"),
+ ],
+ )
+ def test_warns_when_the_aws_connection_credentials_are_not_used(
+ self, aws_base_hook, aws_conn, hook_kwargs, warns
+ ):
+ conn = Connection(conn_id="aws_default", conn_type="aws", **aws_conn)
if aws_conn else None
+ aws_base_hook.conn_config = AwsConnectionWrapper(conn)
+ hook = AwsDuckDBHook(**hook_kwargs)
+ with mock.patch.object(AwsDuckDBHook, "log") as mock_log:
+ statement = secret_statement(hook)
+ secret_statement(hook) # As a second get_conn() would: the
warning is once per hook.
+ assert mock_log.warning.call_args_list == ([mock.call(mock.ANY,
"aws_default")] if warns else [])
+ if "credential_strategy" not in hook_kwargs: # The secret is the same
as before the warning.
+ assert statement == (
+ "CREATE OR REPLACE SECRET airflow_aws "
+ "(TYPE s3, PROVIDER credential_chain, REFRESH 'auto', REGION
'us-west-2');"
+ )
+
def test_credential_chain_does_not_resolve_credentials_itself(self,
aws_base_hook):
"""The point of credential_chain: nothing secret is materialised into
the SQL statement."""
secret_statement(AwsDuckDBHook())
@@ -263,6 +295,16 @@ class TestAwsDuckDBHookConnectionOverrides:
with mock.patch(GET_CONNECTION,
return_value=self.duckdb_connection(credential_strategy="none")):
assert secret_statement(AwsDuckDBHook()) == ""
+ def test_credential_strategy_on_the_connection_silences_the_warning(self,
aws_base_hook):
+ aws_base_hook.conn_config =
AwsConnectionWrapper(Connection(conn_type="aws", **AWS_KEYS))
+ connection =
self.duckdb_connection(credential_strategy="credential_chain")
+ with (
+ mock.patch(GET_CONNECTION, return_value=connection),
+ mock.patch.object(AwsDuckDBHook, "log") as log,
+ ):
+ secret_statement(AwsDuckDBHook())
+ log.warning.assert_not_called()
+
def test_region_can_be_set_on_the_connection(self, aws_base_hook):
with mock.patch(GET_CONNECTION,
return_value=self.duckdb_connection(region_name="ap-south-1")):
assert AwsDuckDBHook().get_region_name() == "ap-south-1"
