sadovnychyi commented on code in PR #33539:
URL: https://github.com/apache/beam/pull/33539#discussion_r1909546321
##########
sdks/python/apache_beam/io/gcp/gcsio.py:
##########
@@ -264,9 +266,45 @@ def delete(self, path):
except NotFound:
return
+ def _batch_with_retry(self, requests, fn):
+ current_requests = [*enumerate(requests)]
+ responses = [None for _ in current_requests]
+
+ @self._storage_client_retry
+ def run_with_retry():
+ current_batch = self.client.batch(raise_exception=False)
+ with current_batch:
+ for _, request in current_requests:
+ fn(request)
+ last_retryable_exception = None
+ for (i, current_pair), response in zip(
+ [*current_requests], current_batch._responses
+ ):
+ responses[i] = response
+ should_retry = (
+ response.status_code >= 400 and
+
self._storage_client_retry._predicate(from_http_response(response)))
+ if should_retry:
+ last_retryable_exception = from_http_response(response)
+ else:
+ current_requests.remove((i, current_pair))
+ if last_retryable_exception:
+ raise last_retryable_exception
+
+ try:
+ run_with_retry()
+ except GoogleCloudError:
Review Comment:
Hmm so I guess I had an expectation that the `retry` object would raise the
last exception which would be one of the HTTP errors, but you are absolutely
right that the `RetryError` is being raised instead.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
