This is an automated email from the ASF dual-hosted git repository.
o-nikolas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new fb959c70741 Skip Azure Blob folder-marker keys in
AzureBlobStorageToS3Operator (#72892)
fb959c70741 is described below
commit fb959c707415a99314ef5123ca65fa2e03f0a4f4
Author: Yuseok Jo <[email protected]>
AuthorDate: Fri Sep 25 01:36:36 2026 +0900
Skip Azure Blob folder-marker keys in AzureBlobStorageToS3Operator (#72892)
Azure container listings include trailing-slash folder markers, which the
operator copied verbatim, leaving spurious empty keys in the destination
bucket and in the XCom output that downstream tasks consume. GCSToS3Operator
already applied an overlap-aware rule for this (#72497), so the two
transfers
disagreed on identical input; both now share it. Sorting the replace=False
path makes the returned key order independent of which keys already exist in
S3.
---
.../amazon/aws/transfers/azure_blob_to_s3.py | 11 +++++-
.../providers/amazon/aws/transfers/gcs_to_s3.py | 26 +------------
.../airflow/providers/amazon/aws/utils/transfer.py | 42 ++++++++++++++++++++
.../amazon/aws/transfers/test_azure_blob_to_s3.py | 21 ++++++++++
.../unit/amazon/aws/transfers/test_gcs_to_s3.py | 26 -------------
.../tests/unit/amazon/aws/utils/test_transfer.py | 45 ++++++++++++++++++++++
6 files changed, 120 insertions(+), 51 deletions(-)
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py
b/providers/amazon/src/airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py
index adb82c42d38..bc2c8bcad6a 100644
---
a/providers/amazon/src/airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py
+++
b/providers/amazon/src/airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py
@@ -23,6 +23,7 @@ from collections.abc import Sequence
from typing import TYPE_CHECKING
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
+from airflow.providers.amazon.aws.utils.transfer import
strip_overlapping_folder_markers
from airflow.providers.common.compat.sdk import BaseOperator
try:
@@ -139,6 +140,14 @@ class AzureBlobStorageToS3Operator(BaseOperator):
container_name=self.container_name, prefix=self.prefix,
endswith=self.delimiter
)
+ files, dropped_keys = strip_overlapping_folder_markers(files)
+ if dropped_keys:
+ self.log.info(
+ "Skipping %s Azure Blob folder-marker key(s) (omitted from
transfer and XCom output): %s",
+ len(dropped_keys),
+ dropped_keys,
+ )
+
if not self.replace:
# if we are not replacing -> list all files in the S3 bucket
# and only keep those files which are present in
@@ -151,7 +160,7 @@ class AzureBlobStorageToS3Operator(BaseOperator):
existing_files = existing_files or []
# remove the prefix for the existing files to allow the match
existing_files = [file.replace(f"{prefix}/", "", 1) for file in
existing_files]
- files = list(set(files) - set(existing_files))
+ files = sorted(set(files) - set(existing_files))
if files:
for file in files:
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py
b/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py
index 85032ecd387..f5cccd2d52e 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py
@@ -26,6 +26,7 @@ from typing import TYPE_CHECKING
from packaging.version import Version
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
+from airflow.providers.amazon.aws.utils.transfer import
strip_overlapping_folder_markers
from airflow.providers.common.compat.sdk import BaseOperator
from airflow.providers.google.cloud.hooks.gcs import GCSHook
@@ -164,29 +165,6 @@ class GCSToS3Operator(BaseOperator):
return os.path.basename(file_path)
return file_path
- @staticmethod
- def _strip_overlapping_folder_markers(keys: list[str]) -> tuple[list[str],
list[str]]:
- """
- Drop trailing-slash keys that are strict prefixes of other listed keys.
-
- Treated as directory markers. A lone trailing-slash key with no overlap
- (e.g. ``lonely/``) is preserved, and a non-slash key that happens to
be a
- strict prefix of another (e.g. ``abc`` of ``abcdef``) is also
preserved.
- Returns ``(kept, dropped)``.
- """
- if not keys:
- return [], []
- ordered = sorted(set(keys))
- kept: list[str] = []
- dropped: list[str] = []
- for current, nxt in zip(ordered, ordered[1:]):
- if current.endswith("/") and nxt.startswith(current):
- dropped.append(current)
- else:
- kept.append(current)
- kept.append(ordered[-1])
- return kept, dropped
-
def execute(self, context: Context) -> list[str]:
# list all files in an Google Cloud Storage bucket
gcs_hook = GCSHook(
@@ -210,7 +188,7 @@ class GCSToS3Operator(BaseOperator):
gcs_files = gcs_hook.list(**list_kwargs) # type: ignore
- gcs_files, dropped_keys =
self._strip_overlapping_folder_markers(gcs_files)
+ gcs_files, dropped_keys = strip_overlapping_folder_markers(gcs_files)
if self.flatten_structure:
# A kept marker like lonely/ has no basename, so flattening would
hit the destination prefix.
dropped_keys += [file for file in gcs_files if not
self._transform_file_path(file)]
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/utils/transfer.py
b/providers/amazon/src/airflow/providers/amazon/aws/utils/transfer.py
new file mode 100644
index 00000000000..474ee5d5627
--- /dev/null
+++ b/providers/amazon/src/airflow/providers/amazon/aws/utils/transfer.py
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Utilities shared by Amazon transfer operators."""
+
+from __future__ import annotations
+
+
+def strip_overlapping_folder_markers(keys: list[str]) -> tuple[list[str],
list[str]]:
+ """
+ Drop trailing-slash keys that are strict prefixes of other listed keys.
+
+ Treated as directory markers. A lone trailing-slash key with no overlap
+ (e.g. ``lonely/``) is preserved, and a non-slash key that happens to be a
+ strict prefix of another (e.g. ``abc`` of ``abcdef``) is also preserved.
+ Returns ``(kept, dropped)``.
+ """
+ if not keys:
+ return [], []
+ ordered = sorted(set(keys))
+ kept: list[str] = []
+ dropped: list[str] = []
+ for current, nxt in zip(ordered, ordered[1:]):
+ if current.endswith("/") and nxt.startswith(current):
+ dropped.append(current)
+ else:
+ kept.append(current)
+ kept.append(ordered[-1])
+ return kept, dropped
diff --git
a/providers/amazon/tests/unit/amazon/aws/transfers/test_azure_blob_to_s3.py
b/providers/amazon/tests/unit/amazon/aws/transfers/test_azure_blob_to_s3.py
index 38db8a87f38..227010c78de 100644
--- a/providers/amazon/tests/unit/amazon/aws/transfers/test_azure_blob_to_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/transfers/test_azure_blob_to_s3.py
@@ -20,6 +20,7 @@ from __future__ import annotations
from io import RawIOBase
from unittest import mock
+import pytest
from moto import mock_aws
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
@@ -216,3 +217,23 @@ class TestAzureBlobToS3Operator:
replace=False,
acl_policy=s3_acl_policy,
)
+
+ @pytest.mark.parametrize("replace", [False, True])
+
@mock.patch("airflow.providers.amazon.aws.transfers.azure_blob_to_s3.WasbHook")
+ def test_execute_skips_overlapping_folder_markers(self, mock_hook,
replace):
+ mock_hook.return_value.get_blobs_list_recursive.return_value = [
+ "src/",
+ "src/file.txt",
+ "lonely/",
+ ]
+ operator = AzureBlobStorageToS3Operator(
+ task_id=TASK_ID,
+ container_name=CONTAINER_NAME,
+ dest_s3_key=S3_BUCKET,
+ replace=replace,
+ )
+ hook, _ = _create_test_bucket()
+
+ uploaded_files = operator.execute(None)
+ assert uploaded_files == ["lonely/", "src/file.txt"]
+ assert hook.list_keys("bucket") == ["lonely/", "src/file.txt"]
diff --git a/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py
b/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py
index feaca455f6b..fd4ead2088f 100644
--- a/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py
@@ -220,32 +220,6 @@ class TestGCSToS3Operator:
assert sorted(MOCK_FILES) == sorted(uploaded_files)
assert sorted(MOCK_FILES) == sorted(hook.list_keys("bucket",
delimiter="/"))
- @pytest.mark.parametrize(
- ("keys", "expected_kept", "expected_dropped"),
- [
- ([], [], []),
- (["a"], ["a"], []),
- (["a", "b"], ["a", "b"], []),
- # Non-slash prefix overlaps must NOT be treated as folder markers.
- (["a", "ax"], ["a", "ax"], []),
- (["abc", "abcdef"], ["abc", "abcdef"], []),
- (["foo/", "foo/bar.txt"], ["foo/bar.txt"], ["foo/"]),
- (
- ["data/", "data/sub/", "data/sub/file.txt"],
- ["data/sub/file.txt"],
- ["data/", "data/sub/"],
- ),
- # A lone trailing-slash key with no overlap is a real object and
stays.
- (["lonely/"], ["lonely/"], []),
- (["lonely/", "report.csv"], ["lonely/", "report.csv"], []),
- ],
- )
- def test_strip_overlapping_folder_markers(self, keys, expected_kept,
expected_dropped):
- """Folder-marker detection: requires both strict-prefix overlap AND
trailing slash."""
- kept, dropped = GCSToS3Operator._strip_overlapping_folder_markers(keys)
- assert kept == expected_kept
- assert dropped == expected_dropped
-
@mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook")
def test_execute_skips_overlapping_folder_markers(self, mock_hook):
mock_hook.return_value.list.return_value = ["src/", "src/airflow.png",
"lonely/"]
diff --git a/providers/amazon/tests/unit/amazon/aws/utils/test_transfer.py
b/providers/amazon/tests/unit/amazon/aws/utils/test_transfer.py
new file mode 100644
index 00000000000..979b99db997
--- /dev/null
+++ b/providers/amazon/tests/unit/amazon/aws/utils/test_transfer.py
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pytest
+
+from airflow.providers.amazon.aws.utils.transfer import
strip_overlapping_folder_markers
+
+
[email protected](
+ ("keys", "expected_kept", "expected_dropped"),
+ [
+ ([], [], []),
+ (["a"], ["a"], []),
+ (["a", "b"], ["a", "b"], []),
+ (["a", "ax"], ["a", "ax"], []),
+ (["abc", "abcdef"], ["abc", "abcdef"], []),
+ (["foo/", "foo/bar.txt"], ["foo/bar.txt"], ["foo/"]),
+ (
+ ["data/", "data/sub/", "data/sub/file.txt"],
+ ["data/sub/file.txt"],
+ ["data/", "data/sub/"],
+ ),
+ (["lonely/"], ["lonely/"], []),
+ (["lonely/", "report.csv"], ["lonely/", "report.csv"], []),
+ ],
+)
+def test_strip_overlapping_folder_markers(keys, expected_kept,
expected_dropped):
+ kept, dropped = strip_overlapping_folder_markers(keys)
+ assert kept == expected_kept
+ assert dropped == expected_dropped
