This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch release-2.54.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.54.0 by this push:
new 8978c8079c2 Support dict as the input of delete_batch (#30241)
8978c8079c2 is described below
commit 8978c8079c2055f4d1448be9ffd68d0f493092fb
Author: Robert Burke <[email protected]>
AuthorDate: Tue Feb 6 16:30:20 2024 -0800
Support dict as the input of delete_batch (#30241)
This smooths the code path of feeding output of list_prefix()
to delete_batch().
Co-authored-by: Shunping Huang <[email protected]>
---
sdks/python/apache_beam/io/gcp/gcsio.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py
b/sdks/python/apache_beam/io/gcp/gcsio.py
index 087d32a9e05..a6ba82a6e07 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -208,7 +208,8 @@ class GcsIO(object):
"""Deletes the objects at the given GCS paths.
Args:
- paths: List of GCS file path patterns in the form gs://<bucket>/<name>,
+ paths: List of GCS file path patterns or Dict with GCS file path patterns
+ as keys. The patterns are in the form gs://<bucket>/<name>, but
not to exceed MAX_BATCH_OPERATION_SIZE in length.
Returns: List of tuples of (path, exception) in the same order as the
@@ -217,6 +218,7 @@ class GcsIO(object):
"""
final_results = []
s = 0
+ if not isinstance(paths, list): paths = list(iter(paths))
while s < len(paths):
if (s + MAX_BATCH_OPERATION_SIZE) < len(paths):
current_paths = paths[s:s + MAX_BATCH_OPERATION_SIZE]
