This is an automated email from the ASF dual-hosted git repository.
vincbeck pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9eb25f92bf2 Add S3ArchiveDagBundle for single-archive staging (#73015)
9eb25f92bf2 is described below
commit 9eb25f92bf25c287ff801d0897c8ee177641710c
Author: Aldo <[email protected]>
AuthorDate: Wed Nov 18 15:25:43 2026 +0100
Add S3ArchiveDagBundle for single-archive staging (#73015)
---
providers/amazon/docs/bundles/index.rst | 41 +++
.../src/airflow/providers/amazon/aws/bundles/s3.py | 306 ++++++++++++++----
.../tests/unit/amazon/aws/bundles/test_s3.py | 356 +++++++++++++++++++--
3 files changed, 621 insertions(+), 82 deletions(-)
diff --git a/providers/amazon/docs/bundles/index.rst
b/providers/amazon/docs/bundles/index.rst
index f332b2f3b73..79986a4a8b6 100644
--- a/providers/amazon/docs/bundles/index.rst
+++ b/providers/amazon/docs/bundles/index.rst
@@ -50,3 +50,44 @@ Example of using the S3DagBundle:
}
}
]'
+
+S3ArchiveDagBundle
+==================
+
+Use :class:`~airflow.providers.amazon.aws.bundles.s3.S3ArchiveDagBundle` when
the Dags are published as one
+``.tar.gz`` object.
:class:`~airflow.providers.amazon.aws.bundles.s3.S3DagBundle` downloads every
object under
+``prefix`` one at a time. Each object costs a full request round-trip, so
bundles made of many small files
+can take a long time to stage — a cost paid by every component that stages the
bundle, which with ephemeral
+workers (e.g. KubernetesExecutor task pods) means on every task start.
+
+``S3ArchiveDagBundle`` stages the bundle from the ``archive_key`` object: one
``HEAD`` request to detect
+changes (unchanged archives are not re-downloaded), one ``GET`` to fetch it,
then a local unpack and a swap
+into place. The archive is the only source of the Dags. Configure this class
instead of ``S3DagBundle``;
+there is no ``prefix`` argument and no second copy to keep in sync.
+
+Archive mode requires Python 3.11.4 or later, because unpacking uses
``tarfile`` ``filter="data"``
+(added in that release). Constructing ``S3ArchiveDagBundle`` on an older
interpreter raises
+``RuntimeError``.
+
+.. code-block:: bash
+
+ export AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST='[
+ {
+ "name": "my-s3-dags",
+ "classpath":
"airflow.providers.amazon.aws.bundles.s3.S3ArchiveDagBundle",
+ "kwargs": {
+ "aws_conn_id": "aws_default",
+ "bucket_name": "my-airflow-bucket",
+ "archive_key": "bundle-archives/dags.tar.gz",
+ "refresh_interval": 60
+ }
+ }
+ ]'
+
+Publishing the archive is the responsibility of your deployment process.
Members are extracted relative to
+the bundle root, so pack the contents of the Dags directory rather than the
directory itself — for example:
+
+.. code-block:: bash
+
+ tar -C ./dags -czf dags.tar.gz .
+ aws s3 cp dags.tar.gz s3://my-airflow-bucket/bundle-archives/dags.tar.gz
diff --git a/providers/amazon/src/airflow/providers/amazon/aws/bundles/s3.py
b/providers/amazon/src/airflow/providers/amazon/aws/bundles/s3.py
index c7f5eb77789..87ac22e189d 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/bundles/s3.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/bundles/s3.py
@@ -17,7 +17,12 @@
from __future__ import annotations
import os
+import shutil
+import tarfile
+import tempfile
+from abc import abstractmethod
from pathlib import Path
+from typing import TYPE_CHECKING
import structlog
@@ -26,47 +31,63 @@ from airflow.providers.amazon.aws.hooks.base_aws import
AwsBaseHook
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
from airflow.providers.common.compat.sdk import AirflowException
+if TYPE_CHECKING:
+ from structlog.typing import FilteringBoundLogger
-class S3DagBundle(BaseDagBundle):
- """
- S3 Dag bundle - exposes a directory in S3 as a Dag bundle.
- This allows Airflow to load Dags directly from an S3 bucket.
+def _require_tar_data_filter() -> None:
+ if hasattr(tarfile, "data_filter"):
+ return
+ raise RuntimeError("Archive Dag bundles require Python 3.11.4 or later
(tarfile filter='data').")
- :param aws_conn_id: Airflow connection ID for AWS. Defaults to
AwsBaseHook.default_conn_name.
- :param bucket_name: The name of the S3 bucket containing the Dag files.
- :param prefix: Optional subdirectory within the S3 bucket where the Dags
are stored.
- A trailing slash is optional.
- If None, Dags are assumed to be at the root of the bucket
(Optional).
+
+def _extract_archive(archive_path: Path, unpack_dir: Path) -> None:
+ """Unpack a bundle archive without writing outside ``unpack_dir``."""
+ with tarfile.open(archive_path, "r:*") as tar:
+ tar.extractall(unpack_dir, filter="data")
+
+
+class _S3DagBundleBase(BaseDagBundle):
"""
+ S3 connection, local directory, and console URL shared by the S3 bundle
classes.
- supports_versioning = False
+ :meta private:
+ """
def __init__(
self,
*,
aws_conn_id: str = AwsBaseHook.default_conn_name,
bucket_name: str,
- prefix: str = "",
**kwargs,
) -> None:
super().__init__(**kwargs)
self.aws_conn_id = aws_conn_id
self.bucket_name = bucket_name
- self.prefix = prefix
# Local path where S3 Dags are downloaded
self.s3_dags_dir: Path = self.base_dir
-
- log = structlog.get_logger(__name__)
- self._log = log.bind(
- bundle_name=self.name,
- version=self.version,
- bucket_name=self.bucket_name,
- prefix=self.prefix,
- aws_conn_id=self.aws_conn_id,
- )
+ self._log = self._get_logger()
self._s3_hook: S3Hook | None = None
+ def _log_bindings(self) -> dict[str, str | None]:
+ """Return log context shared by every S3 Dag bundle."""
+ return {
+ "bundle_name": self.name,
+ "version": self.version,
+ "bucket_name": self.bucket_name,
+ "aws_conn_id": self.aws_conn_id,
+ }
+
+ def _get_logger(self) -> FilteringBoundLogger:
+ """Return a logger bound with the context common to every S3 Dag
bundle."""
+ return structlog.get_logger(__name__).bind(**self._log_bindings())
+
+ def _view_url_key(self) -> str:
+ return ""
+
+ def _validate_remote(self) -> None:
+ """Check the remote source before the first refresh."""
+
def _initialize(self):
with self.lock():
if not self.s3_dags_dir.exists():
@@ -79,14 +100,7 @@ class S3DagBundle(BaseDagBundle):
if not self.s3_hook.check_for_bucket(bucket_name=self.bucket_name):
raise AirflowException(f"S3 bucket '{self.bucket_name}' does
not exist.")
- if self.prefix:
- # don't check when prefix is ""
- if not self.s3_hook.check_for_prefix(
- bucket_name=self.bucket_name, prefix=self.prefix,
delimiter="/"
- ):
- raise AirflowException(
- f"S3 prefix 's3://{self.bucket_name}/{self.prefix}'
does not exist."
- )
+ self._validate_remote()
self.refresh()
def initialize(self) -> None:
@@ -102,16 +116,6 @@ class S3DagBundle(BaseDagBundle):
self._log.warning("Could not create S3Hook for connection %s:
%s", self.aws_conn_id, e)
return self._s3_hook
- def __repr__(self):
- return (
- f"<S3DagBundle("
- f"name={self.name!r}, "
- f"bucket_name={self.bucket_name!r}, "
- f"prefix={self.prefix!r}, "
- f"version={self.version!r}"
- f")>"
- )
-
def get_current_version(self) -> str | None:
"""Return the current version of the Dag bundle. Currently not
supported."""
return None
@@ -121,25 +125,6 @@ class S3DagBundle(BaseDagBundle):
"""Return the local path to the Dag files."""
return self.s3_dags_dir # Path where Dags are downloaded.
- def refresh(self) -> None:
- """Refresh the Dag bundle by re-downloading the Dags from S3."""
- if self.version:
- raise AirflowException("Refreshing a specific version is not
supported")
-
- with self.lock():
- self._log.debug(
- "Downloading Dags from s3://%s/%s to %s", self.bucket_name,
self.prefix, self.s3_dags_dir
- )
- sync_prefix = self.prefix
- if sync_prefix and not sync_prefix.endswith("/"):
- sync_prefix += "/"
- self.s3_hook.sync_to_local_dir(
- bucket_name=self.bucket_name,
- s3_prefix=sync_prefix,
- local_dir=self.s3_dags_dir,
- delete_stale=True,
- )
-
def view_url(self, version: str | None = None) -> str | None:
"""
Return a URL for viewing the Dags in S3. Currently, versioning is not
supported.
@@ -163,7 +148,208 @@ class S3DagBundle(BaseDagBundle):
if self.s3_hook.region_name:
url += f".{self.s3_hook.region_name}"
url += ".amazonaws.com"
- if self.prefix:
- url += f"/{self.prefix}"
+ key = self._view_url_key()
+ if key:
+ url += f"/{key}"
return url
+
+ def refresh(self) -> None:
+ """Refresh the Dag bundle by updating the local Dag files from S3."""
+ if self.version:
+ raise AirflowException("Refreshing a specific version is not
supported")
+ with self.lock():
+ self._refresh_locked()
+
+ @abstractmethod
+ def _refresh_locked(self) -> None:
+ """Stage the bundle while the bundle lock is held."""
+
+
+class S3DagBundle(_S3DagBundleBase):
+ """
+ S3 Dag bundle - exposes a directory in S3 as a Dag bundle.
+
+ This allows Airflow to load Dags directly from an S3 bucket.
+
+ :param aws_conn_id: Airflow connection ID for AWS. Defaults to
AwsBaseHook.default_conn_name.
+ :param bucket_name: The name of the S3 bucket containing the Dag files.
+ :param prefix: Optional subdirectory within the S3 bucket where the Dags
are stored.
+ A trailing slash is optional.
+ If None, Dags are assumed to be at the root of the bucket
(Optional).
+ """
+
+ supports_versioning = False
+
+ def __init__(
+ self,
+ *,
+ aws_conn_id: str = AwsBaseHook.default_conn_name,
+ bucket_name: str,
+ prefix: str = "",
+ **kwargs,
+ ) -> None:
+ self.prefix = prefix
+ super().__init__(aws_conn_id=aws_conn_id, bucket_name=bucket_name,
**kwargs)
+
+ def _get_logger(self) -> FilteringBoundLogger:
+ """Return a logger bound with this prefix bundle's context."""
+ return structlog.get_logger(__name__).bind(**self._log_bindings(),
prefix=self.prefix)
+
+ def _view_url_key(self) -> str:
+ return self.prefix
+
+ def _validate_remote(self) -> None:
+ if self.prefix:
+ # don't check when prefix is ""
+ if not self.s3_hook.check_for_prefix(
+ bucket_name=self.bucket_name, prefix=self.prefix, delimiter="/"
+ ):
+ raise AirflowException(f"S3 prefix
's3://{self.bucket_name}/{self.prefix}' does not exist.")
+
+ def __repr__(self):
+ return (
+ f"<S3DagBundle("
+ f"name={self.name!r}, "
+ f"bucket_name={self.bucket_name!r}, "
+ f"prefix={self.prefix!r}, "
+ f"version={self.version!r}"
+ f")>"
+ )
+
+ def _refresh_locked(self) -> None:
+ self._log.debug(
+ "Downloading Dags from s3://%s/%s to %s", self.bucket_name,
self.prefix, self.s3_dags_dir
+ )
+ sync_prefix = self.prefix
+ if sync_prefix and not sync_prefix.endswith("/"):
+ sync_prefix += "/"
+ self.s3_hook.sync_to_local_dir(
+ bucket_name=self.bucket_name,
+ s3_prefix=sync_prefix,
+ local_dir=self.s3_dags_dir,
+ delete_stale=True,
+ )
+
+
+class S3ArchiveDagBundle(_S3DagBundleBase):
+ """
+ S3 Dag bundle staged from a single ``.tar.gz`` object.
+
+ The archive is the only source of the Dags. Members are extracted relative
to the
+ bundle root, for example ``tar -C <dags_dir> -czf dags.tar.gz .``.
Requires Python
+ 3.11.4 or later.
+
+ :param aws_conn_id: Airflow connection ID for AWS. Defaults to
AwsBaseHook.default_conn_name.
+ :param bucket_name: The name of the S3 bucket containing the archive.
+ :param archive_key: S3 key of the ``.tar.gz`` archive holding the Dag
files.
+ """
+
+ supports_versioning = False
+
+ def __init__(
+ self,
+ *,
+ aws_conn_id: str = AwsBaseHook.default_conn_name,
+ bucket_name: str,
+ archive_key: str,
+ **kwargs,
+ ) -> None:
+ _require_tar_data_filter()
+ self.archive_key = archive_key
+ super().__init__(aws_conn_id=aws_conn_id, bucket_name=bucket_name,
**kwargs)
+ # Keep the Dag tree and its staging dirs under the bundle's own
base_dir.
+ self.s3_dags_dir = self.base_dir / "dags"
+
+ def _get_logger(self) -> FilteringBoundLogger:
+ """Return a logger bound with this archive bundle's context."""
+ return structlog.get_logger(__name__).bind(**self._log_bindings(),
archive_key=self.archive_key)
+
+ def _view_url_key(self) -> str:
+ return self.archive_key
+
+ def __repr__(self):
+ return (
+ f"<S3ArchiveDagBundle("
+ f"name={self.name!r}, "
+ f"bucket_name={self.bucket_name!r}, "
+ f"archive_key={self.archive_key!r}, "
+ f"version={self.version!r}"
+ f")>"
+ )
+
+ def _get_archive_etag_path(self) -> Path:
+ """Return the ETag marker path inside the staged Dag tree."""
+ return self.s3_dags_dir / ".archive_etag"
+
+ def _remove_stale_archive_staging_dirs(self) -> None:
+ if not self.base_dir.is_dir():
+ return
+ for leftover in self.base_dir.iterdir():
+ if leftover.is_dir() and
leftover.name.startswith("archive-staging-"):
+ shutil.rmtree(leftover, ignore_errors=True)
+
+ def _download_archive(self, client, archive_path: Path, etag: str) -> str:
+ """Download the archive object named by ``etag``."""
+ request = {"Bucket": self.bucket_name, "Key": self.archive_key}
+ if etag:
+ request["IfMatch"] = etag
+ response = client.get_object(**request)
+ body = response["Body"]
+ try:
+ with archive_path.open("wb") as handle:
+ shutil.copyfileobj(body, handle, 1024 * 1024)
+ finally:
+ body.close()
+ return response.get("ETag", "")
+
+ def _refresh_locked(self) -> None:
+ self._remove_stale_archive_staging_dirs()
+ client = self.s3_hook.get_conn()
+ head = client.head_object(Bucket=self.bucket_name,
Key=self.archive_key)
+ etag: str = head.get("ETag", "")
+
+ marker = self._get_archive_etag_path()
+ if etag and marker.is_file() and marker.read_text() == etag:
+ self._log.debug(
+ "Dag bundle archive 's3://%s/%s' is unchanged (ETag %s),
skipping staging",
+ self.bucket_name,
+ self.archive_key,
+ etag,
+ )
+ return
+
+ self.base_dir.mkdir(parents=True, exist_ok=True)
+ staging_dir = Path(tempfile.mkdtemp(dir=self.base_dir,
prefix="archive-staging-"))
+ try:
+ archive_path = staging_dir / "_bundle_archive"
+ downloaded_etag = self._download_archive(client, archive_path,
etag)
+
+ unpack_dir = staging_dir / "unpacked"
+ unpack_dir.mkdir()
+ _extract_archive(archive_path, unpack_dir)
+ archive_path.unlink()
+
+ backup_dir = staging_dir / "previous"
+ if self.s3_dags_dir.exists():
+ self.s3_dags_dir.rename(backup_dir)
+ try:
+ unpack_dir.rename(self.s3_dags_dir)
+ except Exception:
+ if backup_dir.exists():
+ backup_dir.rename(self.s3_dags_dir)
+ raise
+
+ marker_etag = downloaded_etag or etag
+ if marker_etag:
+ self._get_archive_etag_path().write_text(marker_etag)
+
+ self._log.debug(
+ "Staged Dag bundle from archive 's3://%s/%s' (%s bytes) to %s",
+ self.bucket_name,
+ self.archive_key,
+ head.get("ContentLength"),
+ self.s3_dags_dir,
+ )
+ finally:
+ shutil.rmtree(staging_dir, ignore_errors=True)
diff --git a/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
b/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
index 66a521a8220..c1719914637 100644
--- a/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
@@ -16,10 +16,16 @@
# under the License.
from __future__ import annotations
+import io
+import shutil
+import tarfile
+from pathlib import Path
+from unittest import mock
from unittest.mock import MagicMock, call
import boto3
import pytest
+from botocore.exceptions import ClientError
from moto import mock_aws
import airflow.version
@@ -34,9 +40,22 @@ AWS_CONN_ID_REGION = "eu-central-1"
AWS_CONN_ID_DEFAULT = "aws_default"
S3_BUCKET_NAME = "my-airflow-dags-bucket"
S3_BUCKET_PREFIX = "project1/dags"
+S3_ARCHIVE_KEY = "bundle-archives/dags.tar.gz"
+
+
+def _make_archive(files: dict[str, bytes]) -> bytes:
+ """Build an in-memory .tar.gz with the given member name -> content
mapping."""
+ buffer = io.BytesIO()
+ with tarfile.open(fileobj=buffer, mode="w:gz") as tar:
+ for name, data in files.items():
+ tar_info = tarfile.TarInfo(name=name)
+ tar_info.size = len(data)
+ tar.addfile(tar_info, io.BytesIO(data))
+ return buffer.getvalue()
+
if airflow.version.version.strip().startswith("3"):
- from airflow.providers.amazon.aws.bundles.s3 import S3DagBundle
+ from airflow.providers.amazon.aws.bundles.s3 import S3ArchiveDagBundle,
S3DagBundle
@pytest.fixture
@@ -73,30 +92,31 @@ def bundle_temp_dir(tmp_path):
yield tmp_path
[email protected](not airflow.version.version.strip().startswith("3"),
reason="Airflow >=3.0.0 test")
-class TestS3DagBundle:
- @pytest.fixture(autouse=True)
- def setup_connections(self, create_connection_without_db):
- create_connection_without_db(
- Connection(
- conn_id=AWS_CONN_ID_DEFAULT,
- conn_type="aws",
- extra={
- "config_kwargs": {"s3": {"bucket_name": S3_BUCKET_NAME}},
- },
- )
[email protected](autouse=True)
+def setup_connections(create_connection_without_db):
+ create_connection_without_db(
+ Connection(
+ conn_id=AWS_CONN_ID_DEFAULT,
+ conn_type="aws",
+ extra={
+ "config_kwargs": {"s3": {"bucket_name": S3_BUCKET_NAME}},
+ },
)
- create_connection_without_db(
- Connection(
- conn_id=AWS_CONN_ID_WITH_REGION,
- conn_type="aws",
- extra={
- "config_kwargs": {"s3": {"bucket_name": S3_BUCKET_NAME}},
- "region_name": AWS_CONN_ID_REGION,
- },
- )
+ )
+ create_connection_without_db(
+ Connection(
+ conn_id=AWS_CONN_ID_WITH_REGION,
+ conn_type="aws",
+ extra={
+ "config_kwargs": {"s3": {"bucket_name": S3_BUCKET_NAME}},
+ "region_name": AWS_CONN_ID_REGION,
+ },
)
+ )
+
[email protected](not airflow.version.version.strip().startswith("3"),
reason="Airflow >=3.0.0 test")
+class TestS3DagBundle:
def test_view_url_generates_presigned_url(self):
bundle = S3DagBundle(
name="test", aws_conn_id=AWS_CONN_ID_DEFAULT,
prefix="project1/dags", bucket_name=S3_BUCKET_NAME
@@ -262,3 +282,295 @@ class TestS3DagBundle:
assert repr(bundle) == original_repr
assert bundle.view_url_template() == original_url
assert s3_client.get_object(Bucket=S3_BUCKET_NAME,
Key=extra_key)["Body"].read() == b"outside"
+
+
[email protected](not airflow.version.version.strip().startswith("3"),
reason="Airflow >=3.0.0 test")
+class TestS3ArchiveDagBundle:
+ def _archive_bundle(self) -> S3ArchiveDagBundle:
+ return S3ArchiveDagBundle(
+ name="test",
+ aws_conn_id=AWS_CONN_ID_WITH_REGION,
+ archive_key=S3_ARCHIVE_KEY,
+ bucket_name=S3_BUCKET_NAME,
+ )
+
+ def test_view_url_template_points_at_archive(self):
+ bundle = self._archive_bundle()
+ assert (
+ bundle.view_url_template()
+ ==
f"https://{S3_BUCKET_NAME}.s3.{AWS_CONN_ID_REGION}.amazonaws.com/{S3_ARCHIVE_KEY}"
+ )
+
+ def test_supports_versioning(self):
+ bundle = self._archive_bundle()
+ assert S3ArchiveDagBundle.supports_versioning is False
+ bundle.version = "test_version"
+ with pytest.raises(AirflowException, match="Refreshing a specific
version is not supported"):
+ bundle.refresh()
+ with pytest.raises(AirflowException, match="S3 url with version is not
supported"):
+ bundle.view_url("test_version")
+
+ def test_refresh_from_archive(self, s3_bucket, s3_client):
+ archive = _make_archive({"dag_01.py": b"test data",
"subproject1/dag_a.py": b"test data"})
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=archive)
+
+ bundle = self._archive_bundle()
+ bundle.initialize()
+
+ assert bundle.path == bundle.base_dir / "dags"
+ assert (bundle.path / "dag_01.py").is_file()
+ assert (bundle.path / "subproject1" / "dag_a.py").is_file()
+ assert bundle._get_archive_etag_path() == bundle.path / ".archive_etag"
+ assert bundle._get_archive_etag_path().is_file()
+ assert sorted(
+ path.relative_to(bundle.path).as_posix()
+ for path in bundle.path.rglob("*")
+ if path != bundle._get_archive_etag_path()
+ ) == [
+ "dag_01.py",
+ "subproject1",
+ "subproject1/dag_a.py",
+ ]
+ assert not (bundle.path / S3_BUCKET_PREFIX / "dag_02.py").exists()
+ assert not (bundle.path / "dag_02.py").exists()
+
+ def test_refresh_from_archive_skips_staging_when_etag_unchanged(self,
s3_bucket, s3_client):
+ archive = _make_archive({"dag_01.py": b"test data"})
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=archive)
+
+ bundle = self._archive_bundle()
+ bundle.initialize()
+
+ sentinel = bundle.path / "sentinel.txt"
+ sentinel.write_text("still here")
+ bundle.refresh()
+ assert sentinel.is_file()
+
+ updated = _make_archive({"dag_new.py": b"test data"})
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=updated)
+ bundle.refresh()
+ assert (bundle.path / "dag_new.py").is_file()
+ assert not (bundle.path / "dag_01.py").exists()
+ assert not sentinel.exists()
+
+ def
test_initialize_restages_when_bundle_dir_cleared_and_etag_matches(self,
s3_bucket, s3_client):
+ archive = _make_archive({"dag_01.py": b"test data"})
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=archive)
+
+ bundle = self._archive_bundle()
+ bundle.initialize()
+ marker = bundle._get_archive_etag_path()
+ etag = marker.read_text()
+ assert marker == bundle.path / ".archive_etag"
+ assert (bundle.path / "dag_01.py").read_bytes() == b"test data"
+
+ shutil.rmtree(bundle.path)
+
+ client = bundle.s3_hook.get_conn()
+ recorded: list[dict] = []
+ original = client.get_object
+
+ def record_get_object(**kwargs):
+ recorded.append(kwargs)
+ return original(**kwargs)
+
+ with mock.patch.object(client, "get_object", record_get_object):
+ bundle.initialize()
+
+ assert recorded
+ assert (bundle.path / "dag_01.py").read_bytes() == b"test data"
+ assert bundle._get_archive_etag_path().read_text() == etag
+
+ def test_refresh_restages_when_bundle_dir_emptied(self, s3_bucket,
s3_client):
+ archive = _make_archive({"dag_01.py": b"test data"})
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=archive)
+
+ bundle = self._archive_bundle()
+ bundle.initialize()
+ for child in bundle.path.iterdir():
+ child.unlink()
+
+ client = bundle.s3_hook.get_conn()
+ recorded: list[dict] = []
+ original = client.get_object
+
+ def record_get_object(**kwargs):
+ recorded.append(kwargs)
+ return original(**kwargs)
+
+ with mock.patch.object(client, "get_object", record_get_object):
+ bundle.refresh()
+
+ assert recorded
+ assert (bundle.path / "dag_01.py").read_bytes() == b"test data"
+
+ def test_prefix_refresh_drops_archive_etag_marker(self, s3_bucket,
s3_client):
+ archive = _make_archive({"archive_only.py": b"from archive"})
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=archive)
+
+ archive_bundle = self._archive_bundle()
+ archive_bundle.initialize()
+ assert (archive_bundle.path / ".archive_etag").is_file()
+ assert (archive_bundle.path / "archive_only.py").is_file()
+
+ prefix_bundle = S3DagBundle(
+ name="test",
+ aws_conn_id=AWS_CONN_ID_WITH_REGION,
+ prefix=S3_BUCKET_PREFIX,
+ bucket_name=S3_BUCKET_NAME,
+ )
+ prefix_bundle.initialize()
+ assert archive_bundle.path == prefix_bundle.path / "dags"
+ assert not (archive_bundle.path / ".archive_etag").exists()
+ assert not (prefix_bundle.path / ".archive_etag").exists()
+ assert (prefix_bundle.path / "dag_01.py").is_file()
+ assert not (prefix_bundle.path / "archive_only.py").exists()
+
+ archive_bundle = self._archive_bundle()
+ archive_bundle.initialize()
+ assert (archive_bundle.path / "archive_only.py").read_bytes() ==
b"from archive"
+ assert not (archive_bundle.path / "dag_01.py").exists()
+ assert (archive_bundle.path / ".archive_etag").is_file()
+
+ def test_initialize_fails_when_archive_is_missing(self, s3_bucket,
s3_client):
+ bundle = self._archive_bundle()
+ with pytest.raises(ClientError) as exc_info:
+ bundle.initialize()
+ assert exc_info.value.response["Error"]["Code"] == "404"
+
+ def test_refresh_raises_and_keeps_bundle_when_archive_is_corrupt(self,
mocked_s3_resource, s3_client):
+ mocked_s3_resource.create_bucket(Bucket=S3_BUCKET_NAME)
+ s3_client.put_object(
+ Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=_make_archive({"dag_01.py": b"test data"})
+ )
+
+ bundle = self._archive_bundle()
+ bundle.initialize()
+ assert (bundle.path / "dag_01.py").is_file()
+
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=b"this is not a tarball")
+ with pytest.raises(tarfile.ReadError):
+ bundle.refresh()
+
+ assert (bundle.path / "dag_01.py").is_file()
+ assert not list(bundle.base_dir.glob("archive-staging-*"))
+
+ def test_archive_staging_restores_previous_bundle_when_swap_fails(self,
mocked_s3_resource, s3_client):
+ mocked_s3_resource.create_bucket(Bucket=S3_BUCKET_NAME)
+ s3_client.put_object(
+ Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=_make_archive({"dag_01.py": b"old"})
+ )
+
+ bundle = self._archive_bundle()
+ bundle.initialize()
+ assert (bundle.path / "dag_01.py").read_text() == "old"
+
+ s3_client.put_object(
+ Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=_make_archive({"dag_01.py": b"new"})
+ )
+
+ real_rename = Path.rename
+
+ def fail_swap_into_place(self: Path, target):
+ if self.name == "unpacked" and Path(target) == bundle.path:
+ raise OSError("swap failed")
+ return real_rename(self, target)
+
+ with (
+ mock.patch.object(Path, "rename", fail_swap_into_place),
+ pytest.raises(OSError, match="swap failed"),
+ ):
+ bundle.refresh()
+
+ assert (bundle.path / "dag_01.py").read_text() == "old"
+ assert not list(bundle.base_dir.glob("archive-staging-*"))
+
+ def test_refresh_removes_only_this_bundles_stale_staging_dir(self,
s3_bucket, s3_client):
+ archive = _make_archive({"dag_01.py": b"test data"})
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=archive)
+ bundle = self._archive_bundle()
+ bundle.initialize()
+
+ storage_root = bundle.base_dir.parent
+ stale = bundle.base_dir / "archive-staging-dead"
+ stale.mkdir()
+ (stale / "orphan.txt").write_text("left by a killed process")
+ dotted_sibling = storage_root / f".{bundle.name}" /
"archive-staging-dead"
+ dotted_sibling.mkdir(parents=True)
+ other = storage_root / "other" / "archive-staging-dead"
+ other.mkdir(parents=True)
+
+ bundle.refresh()
+
+ assert not stale.exists()
+ assert dotted_sibling.is_dir()
+ assert other.is_dir()
+ assert (bundle.path / "dag_01.py").is_file()
+
+ def test_refresh_get_object_uses_head_etag(self, s3_bucket, s3_client):
+ s3_client.put_object(
+ Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=_make_archive({"dag_01.py": b"test data"})
+ )
+ bundle = self._archive_bundle()
+ bundle.initialize()
+ marker = bundle._get_archive_etag_path()
+ etag = marker.read_text()
+ marker.unlink()
+
+ recorded: list[dict] = []
+ client = bundle.s3_hook.get_conn()
+ original = client.get_object
+
+ def record_get_object(**kwargs):
+ recorded.append(kwargs)
+ return original(**kwargs)
+
+ with mock.patch.object(client, "get_object", record_get_object):
+ bundle.refresh()
+
+ assert recorded[0]["IfMatch"] == etag
+ assert (bundle.path / "dag_01.py").is_file()
+
+ def test_refresh_keeps_bundle_when_archive_etag_no_longer_matches(self,
s3_bucket, s3_client):
+ s3_client.put_object(
+ Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=_make_archive({"dag_01.py": b"test data"})
+ )
+ bundle = self._archive_bundle()
+ bundle.initialize()
+ bundle._get_archive_etag_path().unlink()
+
+ error = ClientError(
+ {"Error": {"Code": "PreconditionFailed", "Message": "precondition
failed"}},
+ "GetObject",
+ )
+ client = bundle.s3_hook.get_conn()
+ with mock.patch.object(client, "get_object", side_effect=error),
pytest.raises(ClientError):
+ bundle.refresh()
+
+ assert (bundle.path / "dag_01.py").is_file()
+ assert not list(bundle.base_dir.glob("archive-staging-*"))
+
+ def test_refresh_rejects_archive_member_that_escapes_bundle(self,
mocked_s3_resource, s3_client):
+ mocked_s3_resource.create_bucket(Bucket=S3_BUCKET_NAME)
+ s3_client.put_object(
+ Bucket=S3_BUCKET_NAME, Key=S3_ARCHIVE_KEY,
Body=_make_archive({"dag_01.py": b"kept"})
+ )
+ bundle = self._archive_bundle()
+ bundle.initialize()
+
+ s3_client.put_object(
+ Bucket=S3_BUCKET_NAME,
+ Key=S3_ARCHIVE_KEY,
+ Body=_make_archive({"../escape.py": b"nope"}),
+ )
+ with pytest.raises(tarfile.TarError):
+ bundle.refresh()
+
+ assert (bundle.path / "dag_01.py").read_bytes() == b"kept"
+ assert not list(bundle.base_dir.rglob("escape.py"))
+ assert not list(bundle.base_dir.glob("archive-staging-*"))
+
+ def test_archive_key_requires_tar_data_filter(self, monkeypatch):
+ monkeypatch.delattr(tarfile, "data_filter", raising=False)
+ with pytest.raises(RuntimeError, match="3.11.4"):
+ self._archive_bundle()