Repository: jclouds Updated Branches: refs/heads/master 3e1e9cbab -> 3d508d2d1
Handle HTTP 429 in google-cloud-storage This addresses rateLimitExceeded errors encountered during integration tests. Also increase retry timeout. Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/3d508d2d Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/3d508d2d Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/3d508d2d Branch: refs/heads/master Commit: 3d508d2d165f319eb3b5e8e8179b0e209e61edf7 Parents: 3e1e9cb Author: Andrew Gaul <[email protected]> Authored: Wed Aug 2 22:16:12 2017 -0700 Committer: Andrew Gaul <[email protected]> Committed: Sun Aug 6 23:43:54 2017 -0700 ---------------------------------------------------------------------- .../GoogleCloudStorageApiMetadata.java | 3 ++ .../config/GoogleCloudStorageHttpApiModule.java | 8 ++++ ...ogleCloudStorageClientErrorRetryHandler.java | 49 ++++++++++++++++++++ 3 files changed, 60 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds/blob/3d508d2d/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/GoogleCloudStorageApiMetadata.java ---------------------------------------------------------------------- diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/GoogleCloudStorageApiMetadata.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/GoogleCloudStorageApiMetadata.java index 884edd2..4fc4465 100644 --- a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/GoogleCloudStorageApiMetadata.java +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/GoogleCloudStorageApiMetadata.java @@ -27,6 +27,7 @@ import static org.jclouds.reflect.Reflection2.typeToken; import java.net.URI; import java.util.Properties; +import org.jclouds.Constants; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.googlecloud.config.CurrentProject; import org.jclouds.googlecloudstorage.blobstore.config.GoogleCloudStorageBlobStoreContextModule; @@ -62,6 +63,8 @@ public class GoogleCloudStorageApiMetadata extends BaseHttpApiMetadata<GoogleClo properties.put(OPERATION_COMPLETE_INTERVAL, 2000); properties.put(OPERATION_COMPLETE_TIMEOUT, 600000); properties.setProperty(PROPERTY_IDEMPOTENT_METHODS, "DELETE,GET,HEAD,OPTIONS,POST,PUT"); + // bucket operations have a longer timeout + properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, String.valueOf(TimeUnit.SECONDS.toMillis(1))); return properties; } http://git-wip-us.apache.org/repos/asf/jclouds/blob/3d508d2d/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/config/GoogleCloudStorageHttpApiModule.java ---------------------------------------------------------------------- diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/config/GoogleCloudStorageHttpApiModule.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/config/GoogleCloudStorageHttpApiModule.java index 862f38f..63fec31 100644 --- a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/config/GoogleCloudStorageHttpApiModule.java +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/config/GoogleCloudStorageHttpApiModule.java @@ -22,7 +22,9 @@ import org.jclouds.domain.Credentials; import org.jclouds.googlecloud.config.CurrentProject; import org.jclouds.googlecloudstorage.GoogleCloudStorageApi; import org.jclouds.googlecloudstorage.handlers.GoogleCloudStorageErrorHandler; +import org.jclouds.googlecloudstorage.handlers.GoogleCloudStorageClientErrorRetryHandler; import org.jclouds.http.HttpErrorHandler; +import org.jclouds.http.HttpRetryHandler; import org.jclouds.http.annotation.ClientError; import org.jclouds.http.annotation.Redirection; import org.jclouds.http.annotation.ServerError; @@ -51,6 +53,12 @@ public class GoogleCloudStorageHttpApiModule extends HttpApiModule<GoogleCloudSt bind(HttpErrorHandler.class).annotatedWith(ServerError.class).to(GoogleCloudStorageErrorHandler.class); } + @Override + protected void bindRetryHandlers() { + bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(GoogleCloudStorageClientErrorRetryHandler.class); + // TODO: GoogleCloudStorageRedirectRetryHandler? + } + @Provides @Singleton @CurrentProject public Supplier<String> supplyProject(@Provider final Supplier<Credentials> creds) { http://git-wip-us.apache.org/repos/asf/jclouds/blob/3d508d2d/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageClientErrorRetryHandler.java ---------------------------------------------------------------------- diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageClientErrorRetryHandler.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageClientErrorRetryHandler.java new file mode 100644 index 0000000..f47ff4c --- /dev/null +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageClientErrorRetryHandler.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.googlecloudstorage.handlers; + +import javax.inject.Singleton; + +import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpResponse; +import org.jclouds.http.handlers.BackoffLimitedRetryHandler; +import org.jclouds.http.HttpRetryHandler; + +import com.google.inject.Inject; + +@Singleton +public final class GoogleCloudStorageClientErrorRetryHandler implements HttpRetryHandler { + /** The user has sent too many requests in a given amount of time ("rate limiting"). */ + // TODO: remove when upgrading to jax-rs api 2.1 + private static final int TOO_MANY_REQUESTS = 429; + + private final BackoffLimitedRetryHandler backoffHandler; + + @Inject + protected GoogleCloudStorageClientErrorRetryHandler(BackoffLimitedRetryHandler backoffHandler) { + this.backoffHandler = backoffHandler; + } + + @Override + public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) { + if (response.getStatusCode() == TOO_MANY_REQUESTS) { + return backoffHandler.shouldRetryRequest(command, response); + } else { + return false; + } + } +}
