This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new baac5581c6c Stop creating AWS clients when importing the docs
publisher (#76658)
baac5581c6c is described below
commit baac5581c6c4b79edda19dce13e93d6398b42935
Author: Shahar Epstein <[email protected]>
AuthorDate: Fri Nov 13 14:49:11 2026 +0200
Stop creating AWS clients when importing the docs publisher (#76658)
Importing publish_docs_to_s3 made boto3 resolve credentials, so the import
failed on any machine whose AWS profile cannot be loaded. A profile that
uses
`aws login` fails with botocore's MissingDependencyException unless
botocore[crt] is installed, and that stopped
test_release_management_commands.py
from being collected, although none of its tests talk to AWS.
Claude-Session: https://claude.ai/code/session_01BVx6g6v9hW7wi1tV4bqvrp
---
.../src/airflow_breeze/utils/publish_docs_to_s3.py | 28 +++++++++++++---------
dev/breeze/tests/test_publish_docs_to_s3.py | 9 +++++++
2 files changed, 26 insertions(+), 11 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
index e6722be51ef..9f7ae59c949 100644
--- a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
+++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
@@ -21,7 +21,7 @@ import os
import pathlib
import subprocess
import sys
-from functools import cached_property
+from functools import cache, cached_property
import boto3
@@ -46,8 +46,14 @@ NON_SHORT_NAME_PACKAGES = [
]
-s3_client = boto3.client("s3")
-cloudfront_client = boto3.client("cloudfront")
+@cache
+def get_s3_client():
+ return boto3.client("s3")
+
+
+@cache
+def get_cloudfront_client():
+ return boto3.client("cloudfront")
class VersionError:
@@ -140,7 +146,7 @@ class S3DocsPublish:
def doc_exists(self, s3_bucket_doc_location: str) -> bool:
bucket, key = self.get_bucket_key(s3_bucket_doc_location)
- response = s3_client.list_objects_v2(Bucket=bucket, Prefix=key)
+ response = get_s3_client().list_objects_v2(Bucket=bucket, Prefix=key)
return response["KeyCount"] > 0
@@ -264,7 +270,7 @@ class S3DocsPublish:
List 'directories' (prefixes) in an S3 path using boto3.
"""
bucket, prefix = self.get_bucket_key(s3_path.rstrip("/"))
- paginator = s3_client.get_paginator("list_objects_v2")
+ paginator = get_s3_client().get_paginator("list_objects_v2")
result = []
for page in paginator.paginate(Bucket=bucket, Prefix=prefix + "/",
Delimiter="/"):
for cp in page.get("CommonPrefixes", []):
@@ -300,7 +306,7 @@ class S3DocsPublish:
bucket, _ = self.get_bucket_key(self.destination_location)
console_print("[info]Uploading packages-metadata.json to S3\n")
- s3_client.put_object(
+ get_s3_client().put_object(
Bucket=bucket,
Key="manifest/packages-metadata.json",
Body=json.dumps(all_packages_infos, indent=2),
@@ -311,7 +317,7 @@ class S3DocsPublish:
console_print(
f"[info]Invalidating CloudFront cache for the uploaded files:
distribution id {distribution_id}\n"
)
- cloudfront_client.create_invalidation(
+ get_cloudfront_client().create_invalidation(
DistributionId=distribution_id,
InvalidationBatch={
"Paths": {
@@ -399,7 +405,7 @@ class S3DocsPublish:
<body></body>
</html>"""
- s3_client.put_object(
+ get_s3_client().put_object(
Bucket=bucket,
Key=s3_key,
Body=html_body,
@@ -449,7 +455,7 @@ def _publish_single_schema(
destination =
f"{destination_location.rstrip('/')}/{schema_type}/{version}.json"
bucket, key = S3DocsPublish.get_bucket_key(destination)
- already_published = s3_client.list_objects_v2(Bucket=bucket,
Prefix=key).get("KeyCount", 0) > 0
+ already_published = get_s3_client().list_objects_v2(Bucket=bucket,
Prefix=key).get("KeyCount", 0) > 0
if already_published and not overwrite:
console_print(
f"[info]Skipping {destination} — version {version} already
published "
@@ -462,7 +468,7 @@ def _publish_single_schema(
return False
console_print(f"[info]Publishing {schema_type} version {version} to
{destination}\n")
- s3_client.put_object(
+ get_s3_client().put_object(
Bucket=bucket,
Key=key,
Body=schema_path.read_bytes(),
@@ -479,7 +485,7 @@ def _invalidate_schemas_cache(destination_location: str) ->
None:
console_print(
f"[info]Invalidating CloudFront cache for {invalidation_path}:
distribution id {distribution_id}\n"
)
- cloudfront_client.create_invalidation(
+ get_cloudfront_client().create_invalidation(
DistributionId=distribution_id,
InvalidationBatch={
"Paths": {"Quantity": 1, "Items": [invalidation_path]},
diff --git a/dev/breeze/tests/test_publish_docs_to_s3.py
b/dev/breeze/tests/test_publish_docs_to_s3.py
index 08fe58b7f53..3707aa2127f 100644
--- a/dev/breeze/tests/test_publish_docs_to_s3.py
+++ b/dev/breeze/tests/test_publish_docs_to_s3.py
@@ -17,13 +17,22 @@
from __future__ import annotations
+import runpy
from unittest.mock import MagicMock, PropertyMock, mock_open, patch
import pytest
+from airflow_breeze.utils import publish_docs_to_s3
from airflow_breeze.utils.publish_docs_to_s3 import S3DocsPublish, VersionError
+@patch("boto3.client")
+def test_import_creates_no_aws_clients(mock_client):
+ runpy.run_path(publish_docs_to_s3.__file__)
+
+ mock_client.assert_not_called()
+
+
class TestPublishDocsToS3:
def setup_method(self):
self.publish_docs_to_s3 = S3DocsPublish(
