This is an automated email from the ASF dual-hosted git repository.

vincbeck pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 9eb25f92bf2 Add S3ArchiveDagBundle for single-archive staging (#73015)
9eb25f92bf2 is described below

commit 9eb25f92bf25c287ff801d0897c8ee177641710c
Author: Aldo <[email protected]>
AuthorDate: Wed Nov 18 15:25:43 2026 +0100

    Add S3ArchiveDagBundle for single-archive staging (#73015)
---
 providers/amazon/docs/bundles/index.rst            |  41 +++
 .../src/airflow/providers/amazon/aws/bundles/s3.py | 306 ++++++++++++++----
 .../tests/unit/amazon/aws/bundles/test_s3.py       | 356 +++++++++++++++++++--
 3 files changed, 621 insertions(+), 82 deletions(-)

diff --git a/providers/amazon/docs/bundles/index.rst 
b/providers/amazon/docs/bundles/index.rst
index f332b2f3b73..79986a4a8b6 100644
--- a/providers/amazon/docs/bundles/index.rst
+++ b/providers/amazon/docs/bundles/index.rst
@@ -50,3 +50,44 @@ Example of using the S3DagBundle:
         }
       }
     ]'
+
+S3ArchiveDagBundle
+==================
+
+Use :class:`~airflow.providers.amazon.aws.bundles.s3.S3ArchiveDagBundle` when 
the Dags are published as one
+``.tar.gz`` object. 
:class:`~airflow.providers.amazon.aws.bundles.s3.S3DagBundle` downloads every 
object under
+``prefix`` one at a time. Each object costs a full request round-trip, so 
bundles made of many small files
+can take a long time to stage — a cost paid by every component that stages the 
bundle, which with ephemeral
+workers (e.g. KubernetesExecutor task pods) means on every task start.
+
+``S3ArchiveDagBundle`` stages the bundle from the ``archive_key`` object: one 
``HEAD`` request to detect
+changes (unchanged archives are not re-downloaded), one ``GET`` to fetch it, 
then a local unpack and a swap
+into place. The archive is the only source of the Dags. Configure this class 
instead of ``S3DagBundle``;
+there is no ``prefix`` argument and no second copy to keep in sync.
+
+Archive mode requires Python 3.11.4 or later, because unpacking uses 
``tarfile`` ``filter="data"``
+(added in that release). Constructing ``S3ArchiveDagBundle`` on an older 
interpreter raises
+``RuntimeError``.
+
+.. code-block:: bash
+
+    export AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST='[
+      {
+        "name": "my-s3-dags",
+        "classpath": 
"airflow.providers.amazon.aws.bundles.s3.S3ArchiveDagBundle",
+        "kwargs": {
+          "aws_conn_id": "aws_default",
+          "bucket_name": "my-airflow-bucket",
+          "archive_key": "bundle-archives/dags.tar.gz",
+          "refresh_interval": 60
+        }
+      }
+    ]'
+
+Publishing the archive is the responsibility of your deployment process. 
Members are extracted relative to
+the bundle root, so pack the contents of the Dags directory rather than the 
directory itself — for example:
+
+.. code-block:: bash
+
+    tar -C ./dags -czf dags.tar.gz .
+    aws s3 cp dags.tar.gz s3://my-airflow-bucket/bundle-archives/dags.tar.gz
diff --git a/providers/amazon/src/airflow/providers/amazon/aws/bundles/s3.py 
b/providers/amazon/src/airflow/providers/amazon/aws/bundles/s3.py
index c7f5eb77789..87ac22e189d 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/bundles/s3.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/bundles/s3.py
@@ -17,7 +17,12 @@
 from __future__ import annotations
 
 import os
+import shutil
+import tarfile
+import tempfile
+from abc import abstractmethod
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import structlog
 
@@ -26,47 +31,63 @@ from airflow.providers.amazon.aws.hooks.base_aws import 
AwsBaseHook
 from airflow.providers.amazon.aws.hooks.s3 import S3Hook
 from airflow.providers.common.compat.sdk import AirflowException
 
+if TYPE_CHECKING:
+    from structlog.typing import FilteringBoundLogger
 
-class S3DagBundle(BaseDagBundle):
-    """
-    S3 Dag bundle - exposes a directory in S3 as a Dag bundle.
 
-    This allows Airflow to load Dags directly from an S3 bucket.
+def _require_tar_data_filter() -> None:
+    if hasattr(tarfile, "data_filter"):
+        return
+    raise RuntimeError("Archive Dag bundles require Python 3.11.4 or later 
(tarfile filter='data').")
 
-    :param aws_conn_id: Airflow connection ID for AWS.  Defaults to 
AwsBaseHook.default_conn_name.
-    :param bucket_name: The name of the S3 bucket containing the Dag files.
-    :param prefix:  Optional subdirectory within the S3 bucket where the Dags 
are stored.
-                    A trailing slash is optional.
-                    If None, Dags are assumed to be at the root of the bucket 
(Optional).
+
+def _extract_archive(archive_path: Path, unpack_dir: Path) -> None:
+    """Unpack a bundle archive without writing outside ``unpack_dir``."""
+    with tarfile.open(archive_path, "r:*") as tar:
+        tar.extractall(unpack_dir, filter="data")
+
+
+class _S3DagBundleBase(BaseDagBundle):
     """
+    S3 connection, local directory, and console URL shared by the S3 bundle 
classes.
 
-    supports_versioning = False
+    :meta private:
+    """
 
     def __init__(
         self,
         *,
         aws_conn_id: str = AwsBaseHook.default_conn_name,
         bucket_name: str,
-        prefix: str = "",
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
         self.aws_conn_id = aws_conn_id
         self.bucket_name = bucket_name
-        self.prefix = prefix
         # Local path where S3 Dags are downloaded
         self.s3_dags_dir: Path = self.base_dir
-
-        log = structlog.get_logger(__name__)
-        self._log = log.bind(
-            bundle_name=self.name,
-            version=self.version,
-            bucket_name=self.bucket_name,
-            prefix=self.prefix,
-            aws_conn_id=self.aws_conn_id,
-        )
+        self._log = self._get_logger()
         self._s3_hook: S3Hook | None = None
 
+    def _log_bindings(self) -> dict[str, str | None]:
+        """Return log context shared by every S3 Dag bundle."""
+        return {
+            "bundle_name": self.name,
+            "version": self.version,
+            "bucket_name": self.bucket_name,
+            "aws_conn_id": self.aws_conn_id,
+        }
+
+    def _get_logger(self) -> FilteringBoundLogger:
+        """Return a logger bound with the context common to every S3 Dag 
bundle."""
+        return structlog.get_logger(__name__).bind(**self._log_bindings())
+
+    def _view_url_key(self) -> str:
+        return ""
+
+    def _validate_remote(self) -> None:
+        """Check the remote source before the first refresh."""
+
     def _initialize(self):
         with self.lock():
             if not self.s3_dags_dir.exists():
@@ -79,14 +100,7 @@ class S3DagBundle(BaseDagBundle):
             if not self.s3_hook.check_for_bucket(bucket_name=self.bucket_name):
                 raise AirflowException(f"S3 bucket '{self.bucket_name}' does 
not exist.")
 
-            if self.prefix:
-                # don't check when prefix is ""
-                if not self.s3_hook.check_for_prefix(
-                    bucket_name=self.bucket_name, prefix=self.prefix, 
delimiter="/"
-                ):
-                    raise AirflowException(
-                        f"S3 prefix 's3://{self.bucket_name}/{self.prefix}' 
does not exist."
-                    )
+            self._validate_remote()
             self.refresh()
 
     def initialize(self) -> None:
@@ -102,16 +116,6 @@ class S3DagBundle(BaseDagBundle):
                 self._log.warning("Could not create S3Hook for connection %s: 
%s", self.aws_conn_id, e)
         return self._s3_hook
 
-    def __repr__(self):
-        return (
-            f"<S3DagBundle("
-            f"name={self.name!r}, "
-            f"bucket_name={self.bucket_name!r}, "
-            f"prefix={self.prefix!r}, "
-            f"version={self.version!r}"
-            f")>"
-        )
-
     def get_current_version(self) -> str | None:
         """Return the current version of the Dag bundle. Currently not 
supported."""
         return None
@@ -121,25 +125,6 @@ class S3DagBundle(BaseDagBundle):
         """Return the local path to the Dag files."""
         return self.s3_dags_dir  # Path where Dags are downloaded.
 
-    def refresh(self) -> None:
-        """Refresh the Dag bundle by re-downloading the Dags from S3."""
-        if self.version:
-            raise AirflowException("Refreshing a specific version is not 
supported")
-
-        with self.lock():
-            self._log.debug(
-                "Downloading Dags from s3://%s/%s to %s", self.bucket_name, 
self.prefix, self.s3_dags_dir
-            )
-            sync_prefix = self.prefix
-            if sync_prefix and not sync_prefix.endswith("/"):
-                sync_prefix += "/"
-            self.s3_hook.sync_to_local_dir(
-                bucket_name=self.bucket_name,
-                s3_prefix=sync_prefix,
-                local_dir=self.s3_dags_dir,
-                delete_stale=True,
-            )
-
     def view_url(self, version: str | None = None) -> str | None:
         """
         Return a URL for viewing the Dags in S3. Currently, versioning is not 
supported.
@@ -163,7 +148,208 @@ class S3DagBundle(BaseDagBundle):
         if self.s3_hook.region_name:
             url += f".{self.s3_hook.region_name}"
         url += ".amazonaws.com"
-        if self.prefix:
-            url += f"/{self.prefix}"
+        key = self._view_url_key()
+        if key:
+            url += f"/{key}"
 
         return url
+
+    def refresh(self) -> None:
+        """Refresh the Dag bundle by updating the local Dag files from S3."""
+        if self.version:
+            raise AirflowException("Refreshing a specific version is not 
supported")
+        with self.lock():
+            self._refresh_locked()
+
+    @abstractmethod
+    def _refresh_locked(self) -> None:
+        """Stage the bundle while the bundle lock is held."""
+
+
+class S3DagBundle(_S3DagBundleBase):
+    """
+    S3 Dag bundle - exposes a directory in S3 as a Dag bundle.
+
+    This allows Airflow to load Dags directly from an S3 bucket.
+
+    :param aws_conn_id: Airflow connection ID for AWS.  Defaults to 
AwsBaseHook.default_conn_name.
+    :param bucket_name: The name of the S3 bucket containing the Dag files.
+    :param prefix:  Optional subdirectory within the S3 bucket where the Dags 
are stored.
+                    A trailing slash is optional.
+                    If None, Dags are assumed to be at the root of the bucket 
(Optional).
+    """
+
+    supports_versioning = False
+
+    def __init__(
+        self,
+        *,
+        aws_conn_id: str = AwsBaseHook.default_conn_name,
+        bucket_name: str,
+        prefix: str = "",
+        **kwargs,
+    ) -> None:
+        self.prefix = prefix
+        super().__init__(aws_conn_id=aws_conn_id, bucket_name=bucket_name, 
**kwargs)
+
+    def _get_logger(self) -> FilteringBoundLogger:
+        """Return a logger bound with this prefix bundle's context."""
+        return structlog.get_logger(__name__).bind(**self._log_bindings(), 
prefix=self.prefix)
+
+    def _view_url_key(self) -> str:
+        return self.prefix
+
+    def _validate_remote(self) -> None:
+        if self.prefix:
+            # don't check when prefix is ""
+            if not self.s3_hook.check_for_prefix(
+                bucket_name=self.bucket_name, prefix=self.prefix, delimiter="/"
+            ):
+                raise AirflowException(f"S3 prefix 
's3://{self.bucket_name}/{self.prefix}' does not exist.")
+
+    def __repr__(self):
+        return (
+            f"<S3DagBundle("
+            f"name={self.name!r}, "
+            f"bucket_name={self.bucket_name!r}, "
+            f"prefix={self.prefix!r}, "
+            f"version={self.version!r}"
+            f")>"
+        )
+
+    def _refresh_locked(self) -> None:
+        self._log.debug(
+            "Downloading Dags from s3://%s/%s to %s", self.bucket_name, 
self.prefix, self.s3_dags_dir
+        )
+        sync_prefix = self.prefix
+        if sync_prefix and not sync_prefix.endswith("/"):
+            sync_prefix += "/"
+        self.s3_hook.sync_to_local_dir(
+            bucket_name=self.bucket_name,
+            s3_prefix=sync_prefix,
+            local_dir=self.s3_dags_dir,
+            delete_stale=True,
+        )
+
+
+class S3ArchiveDagBundle(_S3DagBundleBase):
+    """
+    S3 Dag bundle staged from a single ``.tar.gz`` object.
+
+    The archive is the only source of the Dags. Members are extracted relative 
to the
+    bundle root, for example ``tar -C <dags_dir> -czf dags.tar.gz .``. 
Requires Python
+    3.11.4 or later.
+
+    :param aws_conn_id: Airflow connection ID for AWS.  Defaults to 
AwsBaseHook.default_conn_name.
+    :param bucket_name: The name of the S3 bucket containing the archive.
+    :param archive_key: S3 key of the ``.tar.gz`` archive holding the Dag 
files.
+    """
+
+    supports_versioning = False
+
+    def __init__(
+        self,
+        *,
+        aws_conn_id: str = AwsBaseHook.default_conn_name,
+        bucket_name: str,
+        archive_key: str,
+        **kwargs,
+    ) -> None:
+        _require_tar_data_filter()
+        self.archive_key = archive_key
+        super().__init__(aws_conn_id=aws_conn_id, bucket_name=bucket_name, 
**kwargs)
+        # Keep the Dag tree and its staging dirs under the bundle's own 
base_dir.
+        self.s3_dags_dir = self.base_dir / "dags"
+
+    def _get_logger(self) -> FilteringBoundLogger:
+        """Return a logger bound with this archive bundle's context."""
+        return structlog.get_logger(__name__).bind(**self._log_bindings(), 
archive_key=self.archive_key)
+
+    def _view_url_key(self) -> str:
+        return self.archive_key
+
+    def __repr__(self):
+        return (
+            f"<S3ArchiveDagBundle("
+            f"name={self.name!r}, "
+            f"bucket_name={self.bucket_name!r}, "
+            f"archive_key={self.archive_key!r}, "
+            f"version={self.version!r}"
+            f")>"
+        )
+
+    def _get_archive_etag_path(self) -> Path:
+        """Return the ETag marker path inside the staged Dag tree."""
+        return self.s3_dags_dir / ".archive_etag"
+
+    def _remove_stale_archive_staging_dirs(self) -> None:
+        if not self.base_dir.is_dir():
+            return
+        for leftover in self.base_dir.iterdir():
+            if leftover.is_dir() and 
leftover.name.startswith("archive-staging-"):
+                shutil.rmtree(leftover, ignore_errors=True)
+
+    def _download_archive(self, client, archive_path: Path, etag: str) -> str:
+        """Download the archive object named by ``etag``."""
+        request = {"Bucket": self.bucket_name, "Key": self.archive_key}
+        if etag:
+            request["IfMatch"] = etag
+        response = client.get_object(**request)
+        body = response["Body"]
+        try:
+            with archive_path.open("wb") as handle:
+                shutil.copyfileobj(body, handle, 1024 * 1024)
+        finally:
+            body.close()
+        return response.get("ETag", "")
+
+    def _refresh_locked(self) -> None:
+        self._remove_stale_archive_staging_dirs()
+        client = self.s3_hook.get_conn()
+        head = client.head_object(Bucket=self.bucket_name, 
Key=self.archive_key)
+        etag: str = head.get("ETag", "")
+
+        marker = self._get_archive_etag_path()
+        if etag and marker.is_file() and marker.read_text() == etag:
+            self._log.debug(
+                "Dag bundle archive 's3://%s/%s' is unchanged (ETag %s), 
skipping staging",
+                self.bucket_name,
+                self.archive_key,
+                etag,
+            )
+            return
+
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+        staging_dir = Path(tempfile.mkdtemp(dir=self.base_dir, 
prefix="archive-staging-"))
+        try:
+            archive_path = staging_dir / "_bundle_archive"
+            downloaded_etag = self._download_archive(client, archive_path, 
etag)
+
+            unpack_dir = staging_dir / "unpacked"
+            unpack_dir.mkdir()
+            _extract_archive(archive_path, unpack_dir)
+            archive_path.unlink()
+
+            backup_dir = staging_dir / "previous"
+            if self.s3_dags_dir.exists():
+                self.s3_dags_dir.rename(backup_dir)
+            try:
+                unpack_dir.rename(self.s3_dags_dir)
+            except Exception:
+                if backup_dir.exists():
+                    backup_dir.rename(self.s3_dags_dir)
+                raise
+
+            marker_etag = downloaded_etag or etag
+            if marker_etag:
+                self._get_archive_etag_path().write_text(marker_etag)
+
+            self._log.debug(
+                "Staged Dag bundle from archive 's3://%s/%s' (%s bytes) to %s",
+                self.bucket_name,
+                self.archive_key,
+                head.get("ContentLength"),
+                self.s3_dags_dir,
+            )
+        finally:
+            shutil.rmtree(staging_dir, ignore_errors=True)
diff --git a/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py 
b/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
index 66a521a8220..c1719914637 100644
--- a/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
@@ -16,10 +16,16 @@
 # under the License.
 from __future__ import annotations
 
+import io
+import shutil
+import tarfile
+from pathlib import Path
+from unittest import mock
 from unittest.mock import MagicMock, call
 
 import boto3
 import pytest
+from botocore.exceptions import ClientError
 from moto import mock_aws
 
 import airflow.version
@@ -34,9 +40,22 @@ AWS_CONN_ID_REGION = "eu-central-1"
 AWS_CONN_ID_DEFAULT = "aws_default"
 S3_BUCKET_NAME = "my-airflow-dags-bucket"
 S3_BUCKET_PREFIX = "project1/dags"
+S3_ARCHIVE_KEY = "bundle-archives/dags.tar.gz"
+
+
+def _make_archive(files: dict[str, bytes]) -> bytes:
+    """Build an in-memory .tar.gz with the given member name -> content 
mapping."""
+    buffer = io.BytesIO()
+    with tarfile.open(fileobj=buffer, mode="w:gz") as tar:
+        for name, data in files.items():
+            tar_info = tarfile.TarInfo(name=name)
+            tar_info.size = len(data)
+            tar.addfile(tar_info, io.BytesIO(data))
+    return buffer.getvalue()
+
 
 if airflow.version.version.strip().startswith("3"):
-    from airflow.providers.amazon.aws.bundles.s3 import S3DagBundle
+    from airflow.providers.amazon.aws.bundles.s3 import S3ArchiveDagBundle, 
S3DagBundle
 
 
 @pytest.fixture
@@ -73,30 +92,31 @@ def bundle_temp_dir(tmp_path):
         yield tmp_path
 
 
[email protected](not airflow.version.version.strip().startswith("3"), 
reason="Airflow >=3.0.0 test")
-class TestS3DagBundle:
-    @pytest.fixture(autouse=True)
-    def setup_connections(self, create_connection_without_db):
-        create_connection_without_db(
-            Connection(
-                conn_id=AWS_CONN_ID_DEFAULT,
-                conn_type="aws",
-                extra={
-                    "config_kwargs": {"s3": {"bucket_name": S3_BUCKET_NAME}},
-                },
-            )
[email protected](autouse=True)
+def setup_connections(create_connection_without_db):
+    create_connection_without_db(
+        Connection(
+            conn_id=AWS_CONN_ID_DEFAULT,
+            conn_type="aws",
+            extra={
+                "config_kwargs": {"s3": {"bucket_name": S3_BUCKET_NAME}},
+            },
         )
-        create_connection_without_db(
-            Connection(
-                conn_id=AWS_CONN_ID_WITH_REGION,
-                conn_type="aws",
-                extra={
-                    "config_kwargs": {"s3": {"bucket_name": S3_BUCKET_NAME}},
-                    "region_name": AWS_CONN_ID_REGION,
-                },
-            )
+    )
+    create_connection_without_db(
+        Connection(
+            conn_id=AWS_CONN_ID_WITH_REGION,
+            conn_type="aws",
+            extra={
+                "config_kwargs": {"s3": {"bucket_name": S3_BUCKET_NAME}},
+                "region_name": AWS_CONN_ID_REGION,
+            },
         )
+    )
+
 
[email protected](not airflow.version.version.strip().startswith("3"), 
reason="Airflow >=3.0.0 test")
+class TestS3DagBundle:
     def test_view_url_generates_presigned_url(self):
         bundle = S3DagBundle(
             name="test", aws_conn_id=AWS_CONN_ID_DEFAULT, 
prefix="project1/dags", bucket_name=S3_BUCKET_NAME
@@ -262,3 +282,295 @@ class TestS3DagBundle:
         assert repr(bundle) == original_repr
         assert bundle.view_url_template() == original_url
         assert s3_client.get_object(Bucket=S3_BUCKET_NAME, 
Key=extra_key)["Body"].read() == b"outside"
+
+
[email protected](not airflow.version.version.strip().startswith("3"), 
reason="Airflow >=3.0.0 test")
+class TestS3ArchiveDagBundle:
+    def _archive_bundle(self) -> S3ArchiveDagBundle:
+        return S3ArchiveDagBundle(
+            name="test",
+            aws_conn_id=AWS_CONN_ID_WITH_REGION,
+            archive_key=S3_ARCHIVE_KEY,
+            bucket_name=S3_BUCKET_NAME,
+        )
+
+    def test_view_url_template_points_at_archive(self):
+        bundle = self._archive_bundle()
+        assert (
+            bundle.view_url_template()
+            == 
f"https://{S3_BUCKET_NAME}.s3.{AWS_CONN_ID_REGION}.amazonaws.com/{S3_ARCHIVE_KEY}";
+        )
+
+    def test_supports_versioning(self):
+        bundle = self._archive_bundle()
+        assert S3ArchiveDagBundle.supports_versioning is False
+        bundle.version = "test_version"
+        with pytest.raises(AirflowException, match="Refreshing a specific 
version is not supported"):
+            bundle.refresh()
+        with pytest.raises(AirflowException, match="S3 url with version is not 
supported"):
+            bundle.view_url("test_version")
+
+    def test_refresh_from_archive(self, s3_bucket, s3_client):
+        archive = _make_archive({"dag_01.py": b"test data", 
"subproject1/dag_a.py": b"test data"})
+        s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=archive)
+
+        bundle = self._archive_bundle()
+        bundle.initialize()
+
+        assert bundle.path == bundle.base_dir / "dags"
+        assert (bundle.path / "dag_01.py").is_file()
+        assert (bundle.path / "subproject1" / "dag_a.py").is_file()
+        assert bundle._get_archive_etag_path() == bundle.path / ".archive_etag"
+        assert bundle._get_archive_etag_path().is_file()
+        assert sorted(
+            path.relative_to(bundle.path).as_posix()
+            for path in bundle.path.rglob("*")
+            if path != bundle._get_archive_etag_path()
+        ) == [
+            "dag_01.py",
+            "subproject1",
+            "subproject1/dag_a.py",
+        ]
+        assert not (bundle.path / S3_BUCKET_PREFIX / "dag_02.py").exists()
+        assert not (bundle.path / "dag_02.py").exists()
+
+    def test_refresh_from_archive_skips_staging_when_etag_unchanged(self, 
s3_bucket, s3_client):
+        archive = _make_archive({"dag_01.py": b"test data"})
+        s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=archive)
+
+        bundle = self._archive_bundle()
+        bundle.initialize()
+
+        sentinel = bundle.path / "sentinel.txt"
+        sentinel.write_text("still here")
+        bundle.refresh()
+        assert sentinel.is_file()
+
+        updated = _make_archive({"dag_new.py": b"test data"})
+        s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=updated)
+        bundle.refresh()
+        assert (bundle.path / "dag_new.py").is_file()
+        assert not (bundle.path / "dag_01.py").exists()
+        assert not sentinel.exists()
+
+    def 
test_initialize_restages_when_bundle_dir_cleared_and_etag_matches(self, 
s3_bucket, s3_client):
+        archive = _make_archive({"dag_01.py": b"test data"})
+        s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=archive)
+
+        bundle = self._archive_bundle()
+        bundle.initialize()
+        marker = bundle._get_archive_etag_path()
+        etag = marker.read_text()
+        assert marker == bundle.path / ".archive_etag"
+        assert (bundle.path / "dag_01.py").read_bytes() == b"test data"
+
+        shutil.rmtree(bundle.path)
+
+        client = bundle.s3_hook.get_conn()
+        recorded: list[dict] = []
+        original = client.get_object
+
+        def record_get_object(**kwargs):
+            recorded.append(kwargs)
+            return original(**kwargs)
+
+        with mock.patch.object(client, "get_object", record_get_object):
+            bundle.initialize()
+
+        assert recorded
+        assert (bundle.path / "dag_01.py").read_bytes() == b"test data"
+        assert bundle._get_archive_etag_path().read_text() == etag
+
+    def test_refresh_restages_when_bundle_dir_emptied(self, s3_bucket, 
s3_client):
+        archive = _make_archive({"dag_01.py": b"test data"})
+        s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=archive)
+
+        bundle = self._archive_bundle()
+        bundle.initialize()
+        for child in bundle.path.iterdir():
+            child.unlink()
+
+        client = bundle.s3_hook.get_conn()
+        recorded: list[dict] = []
+        original = client.get_object
+
+        def record_get_object(**kwargs):
+            recorded.append(kwargs)
+            return original(**kwargs)
+
+        with mock.patch.object(client, "get_object", record_get_object):
+            bundle.refresh()
+
+        assert recorded
+        assert (bundle.path / "dag_01.py").read_bytes() == b"test data"
+
+    def test_prefix_refresh_drops_archive_etag_marker(self, s3_bucket, 
s3_client):
+        archive = _make_archive({"archive_only.py": b"from archive"})
+        s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=archive)
+
+        archive_bundle = self._archive_bundle()
+        archive_bundle.initialize()
+        assert (archive_bundle.path / ".archive_etag").is_file()
+        assert (archive_bundle.path / "archive_only.py").is_file()
+
+        prefix_bundle = S3DagBundle(
+            name="test",
+            aws_conn_id=AWS_CONN_ID_WITH_REGION,
+            prefix=S3_BUCKET_PREFIX,
+            bucket_name=S3_BUCKET_NAME,
+        )
+        prefix_bundle.initialize()
+        assert archive_bundle.path == prefix_bundle.path / "dags"
+        assert not (archive_bundle.path / ".archive_etag").exists()
+        assert not (prefix_bundle.path / ".archive_etag").exists()
+        assert (prefix_bundle.path / "dag_01.py").is_file()
+        assert not (prefix_bundle.path / "archive_only.py").exists()
+
+        archive_bundle = self._archive_bundle()
+        archive_bundle.initialize()
+        assert (archive_bundle.path / "archive_only.py").read_bytes() == 
b"from archive"
+        assert not (archive_bundle.path / "dag_01.py").exists()
+        assert (archive_bundle.path / ".archive_etag").is_file()
+
+    def test_initialize_fails_when_archive_is_missing(self, s3_bucket, 
s3_client):
+        bundle = self._archive_bundle()
+        with pytest.raises(ClientError) as exc_info:
+            bundle.initialize()
+        assert exc_info.value.response["Error"]["Code"] == "404"
+
+    def test_refresh_raises_and_keeps_bundle_when_archive_is_corrupt(self, 
mocked_s3_resource, s3_client):
+        mocked_s3_resource.create_bucket(Bucket=S3_BUCKET_NAME)
+        s3_client.put_object(
+            Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=_make_archive({"dag_01.py": b"test data"})
+        )
+
+        bundle = self._archive_bundle()
+        bundle.initialize()
+        assert (bundle.path / "dag_01.py").is_file()
+
+        s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=b"this is not a tarball")
+        with pytest.raises(tarfile.ReadError):
+            bundle.refresh()
+
+        assert (bundle.path / "dag_01.py").is_file()
+        assert not list(bundle.base_dir.glob("archive-staging-*"))
+
+    def test_archive_staging_restores_previous_bundle_when_swap_fails(self, 
mocked_s3_resource, s3_client):
+        mocked_s3_resource.create_bucket(Bucket=S3_BUCKET_NAME)
+        s3_client.put_object(
+            Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=_make_archive({"dag_01.py": b"old"})
+        )
+
+        bundle = self._archive_bundle()
+        bundle.initialize()
+        assert (bundle.path / "dag_01.py").read_text() == "old"
+
+        s3_client.put_object(
+            Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=_make_archive({"dag_01.py": b"new"})
+        )
+
+        real_rename = Path.rename
+
+        def fail_swap_into_place(self: Path, target):
+            if self.name == "unpacked" and Path(target) == bundle.path:
+                raise OSError("swap failed")
+            return real_rename(self, target)
+
+        with (
+            mock.patch.object(Path, "rename", fail_swap_into_place),
+            pytest.raises(OSError, match="swap failed"),
+        ):
+            bundle.refresh()
+
+        assert (bundle.path / "dag_01.py").read_text() == "old"
+        assert not list(bundle.base_dir.glob("archive-staging-*"))
+
+    def test_refresh_removes_only_this_bundles_stale_staging_dir(self, 
s3_bucket, s3_client):
+        archive = _make_archive({"dag_01.py": b"test data"})
+        s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=archive)
+        bundle = self._archive_bundle()
+        bundle.initialize()
+
+        storage_root = bundle.base_dir.parent
+        stale = bundle.base_dir / "archive-staging-dead"
+        stale.mkdir()
+        (stale / "orphan.txt").write_text("left by a killed process")
+        dotted_sibling = storage_root / f".{bundle.name}" / 
"archive-staging-dead"
+        dotted_sibling.mkdir(parents=True)
+        other = storage_root / "other" / "archive-staging-dead"
+        other.mkdir(parents=True)
+
+        bundle.refresh()
+
+        assert not stale.exists()
+        assert dotted_sibling.is_dir()
+        assert other.is_dir()
+        assert (bundle.path / "dag_01.py").is_file()
+
+    def test_refresh_get_object_uses_head_etag(self, s3_bucket, s3_client):
+        s3_client.put_object(
+            Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=_make_archive({"dag_01.py": b"test data"})
+        )
+        bundle = self._archive_bundle()
+        bundle.initialize()
+        marker = bundle._get_archive_etag_path()
+        etag = marker.read_text()
+        marker.unlink()
+
+        recorded: list[dict] = []
+        client = bundle.s3_hook.get_conn()
+        original = client.get_object
+
+        def record_get_object(**kwargs):
+            recorded.append(kwargs)
+            return original(**kwargs)
+
+        with mock.patch.object(client, "get_object", record_get_object):
+            bundle.refresh()
+
+        assert recorded[0]["IfMatch"] == etag
+        assert (bundle.path / "dag_01.py").is_file()
+
+    def test_refresh_keeps_bundle_when_archive_etag_no_longer_matches(self, 
s3_bucket, s3_client):
+        s3_client.put_object(
+            Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=_make_archive({"dag_01.py": b"test data"})
+        )
+        bundle = self._archive_bundle()
+        bundle.initialize()
+        bundle._get_archive_etag_path().unlink()
+
+        error = ClientError(
+            {"Error": {"Code": "PreconditionFailed", "Message": "precondition 
failed"}},
+            "GetObject",
+        )
+        client = bundle.s3_hook.get_conn()
+        with mock.patch.object(client, "get_object", side_effect=error), 
pytest.raises(ClientError):
+            bundle.refresh()
+
+        assert (bundle.path / "dag_01.py").is_file()
+        assert not list(bundle.base_dir.glob("archive-staging-*"))
+
+    def test_refresh_rejects_archive_member_that_escapes_bundle(self, 
mocked_s3_resource, s3_client):
+        mocked_s3_resource.create_bucket(Bucket=S3_BUCKET_NAME)
+        s3_client.put_object(
+            Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY, 
Body=_make_archive({"dag_01.py": b"kept"})
+        )
+        bundle = self._archive_bundle()
+        bundle.initialize()
+
+        s3_client.put_object(
+            Bucket=S3_BUCKET_NAME,
+            Key=S3_ARCHIVE_KEY,
+            Body=_make_archive({"../escape.py": b"nope"}),
+        )
+        with pytest.raises(tarfile.TarError):
+            bundle.refresh()
+
+        assert (bundle.path / "dag_01.py").read_bytes() == b"kept"
+        assert not list(bundle.base_dir.rglob("escape.py"))
+        assert not list(bundle.base_dir.glob("archive-staging-*"))
+
+    def test_archive_key_requires_tar_data_filter(self, monkeypatch):
+        monkeypatch.delattr(tarfile, "data_filter", raising=False)
+        with pytest.raises(RuntimeError, match="3.11.4"):
+            self._archive_bundle()

Reply via email to