This is an automated email from the ASF dual-hosted git repository.
o-nikolas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6858cb6c3a2 Refresh expired AWS credentials in AwsDuckDBHook (#76086)
6858cb6c3a2 is described below
commit 6858cb6c3a2af9847bfb2199676e09ea6c623edf
Author: Niko Oliveira <[email protected]>
AuthorDate: Wed Nov 4 14:35:29 2026 -0700
Refresh expired AWS credentials in AwsDuckDBHook (#76086)
* Refresh expired AWS credentials in AwsDuckDBHook
The docs said credential_chain secrets refresh expiring
credentials, but the hook created them without REFRESH 'auto'.
For the default chain, DuckDB then keeps the credentials it first
resolved, so a query that outlives a set of temporary credentials
fails partway through.
DuckDB 1.2.0 and 1.2.1 do not support the option. From 1.2.2 to
1.5.5, DuckDB only refreshes when it opens a file to read, so
listing files, reading a file that is already open, and writing
can still fail once the credentials expire.
* Require DuckDB 1.2.2 for AwsDuckDBHook credential refresh
The hook left REFRESH 'auto' out on DuckDB 1.2.0 and 1.2.1, which
reject it, so users on those versions silently lost the refresh.
The docs also promised that a long query survives expiring
credentials. Before DuckDB 1.5.6 only opening a file to read
refreshes them, so listing files, a long read of one file, or a
write can still fail.
---
providers/amazon/docs/index.rst | 2 +-
providers/amazon/docs/operators/duckdb.rst | 11 ++++++++---
providers/amazon/pyproject.toml | 4 +++-
.../amazon/src/airflow/providers/amazon/aws/hooks/duckdb.py | 9 +++++++--
providers/amazon/tests/unit/amazon/aws/hooks/test_duckdb.py | 3 +++
uv.lock | 2 ++
6 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/providers/amazon/docs/index.rst b/providers/amazon/docs/index.rst
index ae58ec04045..9c9685225bc 100644
--- a/providers/amazon/docs/index.rst
+++ b/providers/amazon/docs/index.rst
@@ -198,7 +198,7 @@ Extra Dependencies
``standard`` ``apache-airflow-providers-standard``
``common.ai`` ``apache-airflow-providers-common-ai>=0.11.0``
``common.messaging`` ``apache-airflow-providers-common-messaging>=2.0.0``
-``duckdb`` ``apache-airflow-providers-duckdb>=0.1.0``
+``duckdb`` ``apache-airflow-providers-duckdb>=0.1.0``,
``duckdb>=1.2.2``
``sqlalchemy`` ``sqlalchemy>=1.4.54,!=2.1.0``
====================
======================================================================================================================================================================
diff --git a/providers/amazon/docs/operators/duckdb.rst
b/providers/amazon/docs/operators/duckdb.rst
index 94edba3e95a..2afea6a11a1 100644
--- a/providers/amazon/docs/operators/duckdb.rst
+++ b/providers/amazon/docs/operators/duckdb.rst
@@ -132,9 +132,14 @@ DuckDB how to authenticate, and the
``credential_strategy`` parameter selects ho
``credential_chain`` (default)
Issues ``CREATE SECRET (TYPE s3, PROVIDER credential_chain)``, which
resolves credentials through
- the AWS SDK inside DuckDB's ``aws`` extension. Nothing secret is written
into SQL text and the SDK
- refreshes expiring credentials itself, so a long query does not fail
partway through when a set of
- temporary credentials expires.
+ the AWS SDK inside DuckDB's ``aws`` extension. Nothing secret is written
into SQL text, and the
+ secret is created with ``REFRESH 'auto'``: when S3 rejects expired
credentials, DuckDB resolves the
+ chain again and retries, so a long query does not fail partway through
when a set of temporary
+ credentials expires.
+
+ Before DuckDB 1.5.6, only opening a file to read refreshes the
credentials. Listing files with a
+ glob, reading further into a file that is already open, and writing with
``COPY ... TO`` can
+ still fail once the credentials expire.
From DuckDB 1.4 onward the chain is resolved when the secret is created
rather than when S3 is
first read. If nothing can be resolved, the hook logs a warning and opens
the connection anyway,
diff --git a/providers/amazon/pyproject.toml b/providers/amazon/pyproject.toml
index 7077049c8a4..0c174769aca 100644
--- a/providers/amazon/pyproject.toml
+++ b/providers/amazon/pyproject.toml
@@ -162,7 +162,9 @@ dependencies = [
# DuckDB support is opt-in: the duckdb wheel is around 20MB and irrelevant to
most users of this
# provider, so it must never become a hard dependency here.
"duckdb" = [
- "apache-airflow-providers-duckdb>=0.1.0"
+ "apache-airflow-providers-duckdb>=0.1.0",
+ # AwsDuckDBHook creates its S3 secret with REFRESH 'auto', which needs
DuckDB 1.2.2.
+ "duckdb>=1.2.2",
]
"sqlalchemy" = [
"sqlalchemy>=1.4.54,!=2.1.0",
diff --git a/providers/amazon/src/airflow/providers/amazon/aws/hooks/duckdb.py
b/providers/amazon/src/airflow/providers/amazon/aws/hooks/duckdb.py
index bf91a0d345d..d9c9e87296f 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/hooks/duckdb.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/hooks/duckdb.py
@@ -59,8 +59,11 @@ class AwsDuckDBHook(DuckDBHook):
``credential_chain`` (the default)
Issues ``CREATE SECRET (TYPE s3, PROVIDER credential_chain)``, which
resolves credentials
through the AWS SDK inside DuckDB's ``aws`` extension. Nothing secret
is written into SQL
- text, and because DuckDB holds the resolution itself the SDK refreshes
expiring credentials,
- so a query that outlives a set of temporary credentials does not fail
partway through.
+ text, and the secret is created with ``REFRESH 'auto'``: when S3
rejects expired credentials,
+ DuckDB resolves the chain again and retries, so a query that outlives
a set of temporary
+ credentials does not fail partway through. Before DuckDB 1.5.6, only
opening a file to read
+ refreshes the credentials, so listing files, reading further into a
file that is already
+ open, and writing can still fail once the credentials expire.
``config``
Resolves credentials through
:class:`~airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook`
@@ -274,6 +277,8 @@ class AwsDuckDBHook(DuckDBHook):
clauses = ["PROVIDER credential_chain"]
if self.credential_chain:
clauses.append(f"CHAIN '{self._quote(self.credential_chain)}'")
+ # Without REFRESH, DuckDB keeps the credentials it first resolved
for the life of the secret.
+ clauses.append("REFRESH 'auto'")
return clauses
if strategy == "config":
credentials =
self.aws_hook.get_credentials(region_name=self.get_region_name())
diff --git a/providers/amazon/tests/unit/amazon/aws/hooks/test_duckdb.py
b/providers/amazon/tests/unit/amazon/aws/hooks/test_duckdb.py
index 4ee6d6a8b6e..c64aa0e1f62 100644
--- a/providers/amazon/tests/unit/amazon/aws/hooks/test_duckdb.py
+++ b/providers/amazon/tests/unit/amazon/aws/hooks/test_duckdb.py
@@ -143,6 +143,9 @@ class TestAwsDuckDBHookCredentials:
secret_statement(AwsDuckDBHook())
aws_base_hook.get_credentials.assert_not_called()
+ def test_credential_chain_secret_refreshes_expired_credentials(self,
aws_base_hook):
+ assert "REFRESH 'auto'" in secret_statement(AwsDuckDBHook())
+
def test_explicit_chain_is_passed_through(self, aws_base_hook):
statement =
secret_statement(AwsDuckDBHook(credential_chain="env;instance"))
assert "CHAIN 'env;instance'" in statement
diff --git a/uv.lock b/uv.lock
index c2ef6283ba2..5ad5608ec2b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -3161,6 +3161,7 @@ common-messaging = [
]
duckdb = [
{ name = "apache-airflow-providers-duckdb" },
+ { name = "duckdb" },
]
exasol = [
{ name = "apache-airflow-providers-exasol" },
@@ -3284,6 +3285,7 @@ requires-dist = [
{ name = "aws-msk-iam-sasl-signer-python", marker = "extra == 'msk'",
specifier = ">=1.0.1" },
{ name = "boto3", specifier = ">=1.43.4" },
{ name = "botocore", specifier = ">=1.43.4" },
+ { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=1.2.2" },
{ name = "inflection", specifier = ">=0.5.1" },
{ name = "jmespath", specifier = ">=0.7.0" },
{ name = "jsonpath-ng", specifier = ">=1.5.3" },
