liferoad commented on code in PR #34835:
URL: https://github.com/apache/beam/pull/34835#discussion_r2073538555
##########
sdks/python/apache_beam/io/gcp/gcsio.py:
##########
@@ -265,10 +265,23 @@ def delete(self, path, recursive=False):
bucket_name, blob_name = parse_gcs_path(path)
bucket = self.client.bucket(bucket_name)
if recursive:
- # List and delete all blobs under the prefix.
- blobs = bucket.list_blobs(prefix=blob_name)
- for blob in blobs:
- self._delete_blob(bucket, blob.name)
+ # List all blobs under the prefix.
+ blobs_to_delete = bucket.list_blobs(
+ prefix=blob_name, retry=self._storage_client_retry)
+ # Collect full paths for batch deletion.
+ paths_to_delete = [
+ f'gs://{bucket_name}/{blob.name}' for blob in blobs_to_delete
+ ]
+ if paths_to_delete:
+ # Delete them in batches.
+ results = self.delete_batch(paths_to_delete)
+ # Log any errors encountered during batch deletion.
+ errors = [f'{path}: {err}' for path, err in results if err is not None]
+ if errors:
+ _LOGGER.warning(
+ 'Failed to delete some objects during recursive delete of %s:
%s',
Review Comment:
given `', '.join(errors)` is used here, I think it is good to keep my
current style since it is much easier to read.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@beam.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
