This is an automated email from the ASF dual-hosted git repository.
anton pushed a commit to branch release-2.14.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.14.0 by this push:
new 5208659 [BEAM-7424] Retry HTTP 429 errors from GCS
new 0bdb818 Merge pull request #9014 from ihji/BEAM-7424
5208659 is described below
commit 5208659728734bd7fc991a86d72b80e816c2d915
Author: Heejong Lee <heej...@gmail.com>
AuthorDate: Mon Jun 24 11:55:52 2019 -0700
[BEAM-7424] Retry HTTP 429 errors from GCS
match Python SDK retrying threshold to Java SDK threshold.
---
sdks/python/apache_beam/io/gcp/gcsio.py | 3 ++-
.../apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py
b/sdks/python/apache_beam/io/gcp/gcsio.py
index 5586eb4..6a27c27 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -522,7 +522,8 @@ class GcsDownloader(Downloader):
# Initialize read buffer state.
self._download_stream = io.BytesIO()
self._downloader = transfer.Download(
- self._download_stream, auto_transfer=False,
chunksize=self._buffer_size)
+ self._download_stream, auto_transfer=False,
chunksize=self._buffer_size,
+ num_retries=20)
self._client.objects.Get(self._get_request, download=self._downloader)
@retry.with_exponential_backoff(
diff --git
a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py
b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py
index cb5a0c9..735b159 100644
---
a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py
+++
b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py
@@ -52,7 +52,7 @@ class StorageV1(base_api.BaseApiClient):
super(StorageV1, self).__init__(
url, credentials=credentials,
get_credentials=get_credentials, http=http, model=model,
- log_request=log_request, log_response=log_response,
+ log_request=log_request, log_response=log_response, num_retries=20,
credentials_args=credentials_args,
default_global_params=default_global_params,
additional_http_headers=additional_http_headers,
