Repository: jclouds-labs Updated Branches: refs/heads/master 52e8f9e44 -> 436c001fb
RateLimit module for ProfitBricks REST Added a module that automatically deals with rate limits in the ProfitBricks REST api and throttles the requests as needed. The module is configured by default in the live tests, since I've observed several failures due to rate limits when using my account, but not added by default. Users that want jclouds to deal automatically with this will have to register the module when creating the context. Project: http://git-wip-us.apache.org/repos/asf/jclouds-labs/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds-labs/commit/436c001f Tree: http://git-wip-us.apache.org/repos/asf/jclouds-labs/tree/436c001f Diff: http://git-wip-us.apache.org/repos/asf/jclouds-labs/diff/436c001f Branch: refs/heads/master Commit: 436c001fbc10857104cd1c4a8f74da64a18bd74d Parents: 52e8f9e Author: Ignasi Barrera <[email protected]> Authored: Wed Oct 26 15:35:40 2016 +0200 Committer: Ignasi Barrera <[email protected]> Committed: Wed Oct 26 15:35:40 2016 +0200 ---------------------------------------------------------------------- .../config/ProfitBricksRateLimitModule.java | 30 ++++++++ .../ProfitBricksRateLimitExceededException.java | 79 ++++++++++++++++++++ .../handlers/ProfitBricksHttpErrorHandler.java | 7 +- .../ProfitBricksRateLimitRetryHandler.java | 39 ++++++++++ .../ProfitBricksComputeServiceLiveTest.java | 10 +++ .../ProfitBricksTemplateBuilderLiveTest.java | 11 +++ .../rest/internal/BaseProfitBricksLiveTest.java | 8 ++ 7 files changed, 181 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/config/ProfitBricksRateLimitModule.java ---------------------------------------------------------------------- diff --git a/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/config/ProfitBricksRateLimitModule.java b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/config/ProfitBricksRateLimitModule.java new file mode 100644 index 0000000..3f100e4 --- /dev/null +++ b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/config/ProfitBricksRateLimitModule.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jclouds.profitbricks.rest.config; + +import org.apache.jclouds.profitbricks.rest.handlers.ProfitBricksRateLimitRetryHandler; +import org.jclouds.http.HttpRetryHandler; +import org.jclouds.http.annotation.ClientError; + +import com.google.inject.AbstractModule; + +public class ProfitBricksRateLimitModule extends AbstractModule { + @Override + protected void configure() { + bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(ProfitBricksRateLimitRetryHandler.class); + } +} http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/exceptions/ProfitBricksRateLimitExceededException.java ---------------------------------------------------------------------- diff --git a/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/exceptions/ProfitBricksRateLimitExceededException.java b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/exceptions/ProfitBricksRateLimitExceededException.java new file mode 100644 index 0000000..5f0af6f --- /dev/null +++ b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/exceptions/ProfitBricksRateLimitExceededException.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jclouds.profitbricks.rest.exceptions; + +import org.jclouds.http.HttpResponse; +import org.jclouds.rest.RateLimitExceededException; + +import com.google.common.annotations.Beta; +import com.google.common.base.Predicate; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; + +/** + * Provides detailed information for rate limit exceptions. + */ +@Beta +public class ProfitBricksRateLimitExceededException extends RateLimitExceededException { + private static final long serialVersionUID = 1L; + private static final String RATE_LIMIT_HEADER_PREFIX = "X-RateLimit-"; + + private Integer maxConcurrentRequestsAllowed; + private Integer remainingRequests; + private Integer averageRequestsAllowedPerMinute; + + public ProfitBricksRateLimitExceededException(HttpResponse response) { + super(response.getStatusLine() + "\n" + rateLimitHeaders(response)); + parseRateLimitInfo(response); + } + + public ProfitBricksRateLimitExceededException(HttpResponse response, Throwable cause) { + super(response.getStatusLine() + "\n" + rateLimitHeaders(response), cause); + parseRateLimitInfo(response); + } + + public Integer maxConcurrentRequestsAllowed() { + return maxConcurrentRequestsAllowed; + } + + public Integer remainingRequests() { + return remainingRequests; + } + + public Integer averageRequestsAllowedPerMinute() { + return averageRequestsAllowedPerMinute; + } + + private void parseRateLimitInfo(HttpResponse response) { + String burst = response.getFirstHeaderOrNull("X-RateLimit-Burst"); + String remaining = response.getFirstHeaderOrNull("X-RateLimit-Remaining"); + String limit = response.getFirstHeaderOrNull("X-RateLimit-Limit"); + + maxConcurrentRequestsAllowed = burst == null ? null : Integer.valueOf(burst); + remainingRequests = remaining == null ? null : Integer.valueOf(remaining); + averageRequestsAllowedPerMinute = limit == null ? null : Integer.valueOf(limit); + } + + private static Multimap<String, String> rateLimitHeaders(HttpResponse response) { + return Multimaps.filterKeys(response.getHeaders(), new Predicate<String>() { + @Override + public boolean apply(String input) { + return input.startsWith(RATE_LIMIT_HEADER_PREFIX); + } + }); + } +} http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksHttpErrorHandler.java ---------------------------------------------------------------------- diff --git a/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksHttpErrorHandler.java b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksHttpErrorHandler.java index 188c900..6c46cad 100644 --- a/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksHttpErrorHandler.java +++ b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksHttpErrorHandler.java @@ -16,10 +16,9 @@ */ package org.apache.jclouds.profitbricks.rest.handlers; -import static org.jclouds.util.Closeables2.closeQuietly; - import javax.inject.Singleton; +import org.apache.jclouds.profitbricks.rest.exceptions.ProfitBricksRateLimitExceededException; import org.jclouds.http.HttpCommand; import org.jclouds.http.HttpErrorHandler; import org.jclouds.http.HttpResponse; @@ -58,6 +57,9 @@ public class ProfitBricksHttpErrorHandler implements HttpErrorHandler { if (!command.getCurrentRequest().getMethod().equals("DELETE")) exception = new ResourceNotFoundException(response.getMessage(), exception); break; + case 429: + exception = new ProfitBricksRateLimitExceededException(response); + break; case 413: case 503: // if nothing (default message was OK) was parsed from command executor, assume it was an 503 (Maintenance) html response. @@ -68,7 +70,6 @@ public class ProfitBricksHttpErrorHandler implements HttpErrorHandler { break; } } finally { - closeQuietly(response.getPayload()); command.setException(exception); } } http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksRateLimitRetryHandler.java ---------------------------------------------------------------------- diff --git a/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksRateLimitRetryHandler.java b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksRateLimitRetryHandler.java new file mode 100644 index 0000000..9aef958 --- /dev/null +++ b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksRateLimitRetryHandler.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jclouds.profitbricks.rest.handlers; + +import javax.inject.Singleton; + +import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpResponse; +import org.jclouds.http.handlers.RateLimitRetryHandler; + +import com.google.common.annotations.Beta; +import com.google.common.base.Optional; +import com.google.common.net.HttpHeaders; + +@Beta +@Singleton +public class ProfitBricksRateLimitRetryHandler extends RateLimitRetryHandler { + + @Override + protected Optional<Long> millisToNextAvailableRequest(HttpCommand command, HttpResponse response) { + String secondsToNextAvailableRequest = response.getFirstHeaderOrNull(HttpHeaders.RETRY_AFTER); + return secondsToNextAvailableRequest != null ? Optional.of(Long.valueOf(secondsToNextAvailableRequest) * 1000) + : Optional.<Long> absent(); + } +} http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksComputeServiceLiveTest.java ---------------------------------------------------------------------- diff --git a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksComputeServiceLiveTest.java b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksComputeServiceLiveTest.java index de6a2fb..a3a15b3 100644 --- a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksComputeServiceLiveTest.java +++ b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksComputeServiceLiveTest.java @@ -26,6 +26,8 @@ import org.jclouds.compute.domain.NodeMetadata; import org.jclouds.compute.domain.Template; import org.jclouds.compute.internal.BaseComputeServiceLiveTest; import static org.jclouds.compute.predicates.NodePredicates.inGroup; + +import org.apache.jclouds.profitbricks.rest.config.ProfitBricksRateLimitModule; import org.jclouds.logging.config.LoggingModule; import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; import org.jclouds.sshj.config.SshjSshClientModule; @@ -47,6 +49,14 @@ public class ProfitBricksComputeServiceLiveTest extends BaseComputeServiceLiveTe protected LoggingModule getLoggingModule() { return new SLF4JLoggingModule(); } + + @Override + protected Iterable<Module> setupModules() { + ImmutableSet.Builder<Module> modules = ImmutableSet.builder(); + modules.addAll(super.setupModules()); + modules.add(new ProfitBricksRateLimitModule()); + return modules.build(); + } @Override public void testOptionToNotBlock() throws Exception { http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksTemplateBuilderLiveTest.java ---------------------------------------------------------------------- diff --git a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksTemplateBuilderLiveTest.java b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksTemplateBuilderLiveTest.java index afe1be2..f029c41 100644 --- a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksTemplateBuilderLiveTest.java +++ b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksTemplateBuilderLiveTest.java @@ -17,7 +17,11 @@ package org.apache.jclouds.profitbricks.rest.compute; import com.google.common.collect.ImmutableSet; +import com.google.inject.Module; + import java.util.Set; + +import org.apache.jclouds.profitbricks.rest.config.ProfitBricksRateLimitModule; import org.jclouds.compute.internal.BaseTemplateBuilderLiveTest; import org.testng.annotations.Test; @@ -33,4 +37,11 @@ public class ProfitBricksTemplateBuilderLiveTest extends BaseTemplateBuilderLive return ImmutableSet.of("DE-BW", "DE-HE", "US-NV"); } + @Override + protected Iterable<Module> setupModules() { + ImmutableSet.Builder<Module> modules = ImmutableSet.builder(); + modules.addAll(super.setupModules()); + modules.add(new ProfitBricksRateLimitModule()); + return modules.build(); + } } http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/internal/BaseProfitBricksLiveTest.java ---------------------------------------------------------------------- diff --git a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/internal/BaseProfitBricksLiveTest.java b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/internal/BaseProfitBricksLiveTest.java index 9e50dba..e1c3145 100644 --- a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/internal/BaseProfitBricksLiveTest.java +++ b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/internal/BaseProfitBricksLiveTest.java @@ -18,6 +18,7 @@ package org.apache.jclouds.profitbricks.rest.internal; import com.google.common.base.Joiner; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; @@ -27,6 +28,8 @@ import java.util.Properties; import java.util.concurrent.TimeUnit; import org.apache.jclouds.profitbricks.rest.ProfitBricksApi; import org.apache.jclouds.profitbricks.rest.compute.config.ProfitBricksComputeServiceContextModule.ComputeConstants; +import org.apache.jclouds.profitbricks.rest.config.ProfitBricksRateLimitModule; + import static org.apache.jclouds.profitbricks.rest.config.ProfitBricksComputeProperties.POLL_PREDICATE_DATACENTER; import org.apache.jclouds.profitbricks.rest.domain.DataCenter; import org.apache.jclouds.profitbricks.rest.domain.LicenceType; @@ -61,6 +64,11 @@ public class BaseProfitBricksLiveTest extends BaseApiLiveTest<ProfitBricksApi> { public BaseProfitBricksLiveTest() { provider = "profitbricks-rest"; } + + @Override + protected Iterable<Module> setupModules() { + return ImmutableSet.<Module> of(getLoggingModule(), new ProfitBricksRateLimitModule()); + } @Override protected ProfitBricksApi create(Properties props, Iterable<Module> modules) {
