This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c5c50cc07f Revert "Fix `GCSToGCSOperator` behavior difference for
moving single object (#40162)" (#40577)
c5c50cc07f is described below
commit c5c50cc07f6fcd704981139beb54095d8b9938c7
Author: Maksim <[email protected]>
AuthorDate: Wed Jul 3 02:45:56 2024 -0700
Revert "Fix `GCSToGCSOperator` behavior difference for moving single object
(#40162)" (#40577)
This reverts commit 2f2796f675e238e0b266b8d6e4fdfe5c0b7d1bf3.
---
.../providers/google/cloud/transfers/gcs_to_gcs.py | 18 ++++++---
.../google/cloud/transfers/test_gcs_to_gcs.py | 46 ----------------------
2 files changed, 12 insertions(+), 52 deletions(-)
diff --git a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py
b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py
index 7fb12dbf09..0b3d330b65 100644
--- a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py
+++ b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py
@@ -408,9 +408,20 @@ class GCSToGCSOperator(BaseOperator):
msg = f"{prefix} does not exist in bucket {self.source_bucket}"
self.log.warning(msg)
raise AirflowException(msg)
+ if len(objects) == 1 and objects[0][-1] != "/":
+ self._copy_file(hook=hook, source_object=objects[0])
elif len(objects):
self._copy_multiple_objects(hook=hook, source_objects=objects,
prefix=prefix)
+ def _copy_file(self, hook, source_object):
+ destination_object = self.destination_object or source_object
+ if self.destination_object and self.destination_object[-1] == "/":
+ file_name = source_object.split("/")[-1]
+ destination_object += file_name
+ self._copy_single_object(
+ hook=hook, source_object=source_object,
destination_object=destination_object
+ )
+
def _copy_multiple_objects(self, hook, source_objects, prefix):
# Check whether the prefix is a root directory for all the rest of
objects.
_pref = prefix.rstrip("/")
@@ -430,12 +441,7 @@ class GCSToGCSOperator(BaseOperator):
destination_object = source_obj
else:
file_name_postfix = source_obj.replace(base_path, "", 1)
-
- destination_object = (
- self.destination_object.rstrip("/")[0 :
self.destination_object.rfind("/")]
- + "/"
- + file_name_postfix
- )
+ destination_object = self.destination_object.rstrip("/") + "/"
+ file_name_postfix
self._copy_single_object(
hook=hook, source_object=source_obj,
destination_object=destination_object
diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py
b/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py
index 4f63326af7..97e16e7366 100644
--- a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py
+++ b/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py
@@ -742,52 +742,6 @@ class TestGoogleCloudStorageToCloudStorageOperator:
["source/foo.txt"],
["{prefix}/foo.txt"],
),
- (
- ["source/sub1/sub2/sub3/file.txt"],
- "source/",
- None,
- False,
- ["source/sub1/sub2/sub3/file.txt"],
- ["{prefix}/sub1/sub2/sub3/file.txt"],
- ),
- (
- ["source/sub1/sub2/sub3/file.txt",
"source/sub1/sub2/sub3/file2.txt"],
- "source/",
- None,
- False,
- ["source/sub1/sub2/sub3/file.txt",
"source/sub1/sub2/sub3/file2.txt"],
- ["{prefix}/sub1/sub2/sub3/file.txt",
"{prefix}/sub1/sub2/sub3/file2.txt"],
- ),
- (
- [f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt"],
- f"{DESTINATION_OBJECT_PREFIX}",
- None,
- False,
- [f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt"],
- ["{prefix}/sub1/sub2/sub3/file.txt"],
- ),
- (
- [f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt"],
- f"{DESTINATION_OBJECT_PREFIX}/",
- None,
- False,
- [f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt"],
- ["{prefix}/sub1/sub2/sub3/file.txt"],
- ),
- (
- [
- f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt",
- f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file2.txt",
- ],
- f"{DESTINATION_OBJECT_PREFIX}/",
- None,
- False,
- [
- f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt",
- f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file2.txt",
- ],
- ["{prefix}/sub1/sub2/sub3/file.txt",
"{prefix}/sub1/sub2/sub3/file2.txt"],
- ),
(
["source/foo.txt", "source/foo.txt.abc",
"source/foo.txt/subfolder/file.txt"],
"source/foo.txt",
