JCLOUDS-1022: Automatically handle DigitalOcean rate limit
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/7e866ad6 Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/7e866ad6 Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/7e866ad6 Branch: refs/heads/master Commit: 7e866ad6a134357ce34ba7ae5247a088aac5c79e Parents: 4596471 Author: Ignasi Barrera <[email protected]> Authored: Thu Oct 22 00:31:58 2015 +0200 Committer: Ignasi Barrera <[email protected]> Committed: Tue Oct 27 23:30:43 2015 +0100 ---------------------------------------------------------------------- .../config/DigitalOcean2Properties.java | 33 ++++ .../config/DigitalOcean2RateLimitModule.java | 30 ++++ ...DigitalOcean2RateLimitExceededException.java | 81 ++++++++++ .../handlers/DigitalOcean2ErrorHandler.java | 9 +- .../handlers/RateLimitRetryHandler.java | 111 ++++++++++++++ .../DigitalOcean2ComputeServiceLiveTest.java | 8 + .../exceptions/RateLimitExceptionMockTest.java | 63 ++++++++ .../handlers/RateLimitRetryHandlerTest.java | 153 +++++++++++++++++++ .../internal/BaseDigitalOcean2ApiLiveTest.java | 7 + .../internal/BaseDigitalOcean2ApiMockTest.java | 7 +- 10 files changed, 499 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds/blob/7e866ad6/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2Properties.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2Properties.java b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2Properties.java new file mode 100644 index 0000000..d0d1098 --- /dev/null +++ b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2Properties.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.digitalocean2.config; + +public final class DigitalOcean2Properties { + + /** + * Maximum amount of time (in milliseconds) a request will wait until retrying if + * the rate limit is exhausted. + * <p> + * Default value: 2 minutes. + */ + public static final String MAX_RATE_LIMIT_WAIT = "jclouds.max-ratelimit-wait"; + + private DigitalOcean2Properties() { + throw new AssertionError("intentionally unimplemented"); + } + +} http://git-wip-us.apache.org/repos/asf/jclouds/blob/7e866ad6/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2RateLimitModule.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2RateLimitModule.java b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2RateLimitModule.java new file mode 100644 index 0000000..1b0a95f --- /dev/null +++ b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2RateLimitModule.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.digitalocean2.config; + +import org.jclouds.digitalocean2.handlers.RateLimitRetryHandler; +import org.jclouds.http.HttpRetryHandler; +import org.jclouds.http.annotation.ClientError; + +import com.google.inject.AbstractModule; + +public class DigitalOcean2RateLimitModule extends AbstractModule { + @Override + protected void configure() { + bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(RateLimitRetryHandler.class); + } +} http://git-wip-us.apache.org/repos/asf/jclouds/blob/7e866ad6/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/exceptions/DigitalOcean2RateLimitExceededException.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/exceptions/DigitalOcean2RateLimitExceededException.java b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/exceptions/DigitalOcean2RateLimitExceededException.java new file mode 100644 index 0000000..fc54a7c --- /dev/null +++ b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/exceptions/DigitalOcean2RateLimitExceededException.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.digitalocean2.exceptions; + +import static org.jclouds.digitalocean2.handlers.RateLimitRetryHandler.millisUntilNextAvailableRequest; + +import org.jclouds.http.HttpResponse; +import org.jclouds.rest.RateLimitExceededException; + +import com.google.common.annotations.Beta; +import com.google.common.base.Predicate; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; + +/** + * Provides detailed information for rate limit exceptions. + */ +@Beta +public class DigitalOcean2RateLimitExceededException extends RateLimitExceededException { + private static final long serialVersionUID = 1L; + private static final String RATE_LIMIT_HEADER_PREFIX = "RateLimit-"; + + private Integer totalRequestsPerHour; + private Integer remainingRequests; + private Long timeToNextAvailableRequest; + + public DigitalOcean2RateLimitExceededException(HttpResponse response) { + super(response.getStatusLine() + "\n" + rateLimitHeaders(response)); + parseRateLimitInfo(response); + } + + public DigitalOcean2RateLimitExceededException(HttpResponse response, Throwable cause) { + super(response.getStatusLine() + "\n" + rateLimitHeaders(response), cause); + parseRateLimitInfo(response); + } + + public Integer totalRequestsPerHour() { + return totalRequestsPerHour; + } + + public Integer remainingRequests() { + return remainingRequests; + } + + public Long timeToNextAvailableRequest() { + return timeToNextAvailableRequest; + } + + private void parseRateLimitInfo(HttpResponse response) { + String limit = response.getFirstHeaderOrNull("RateLimit-Limit"); + String remaining = response.getFirstHeaderOrNull("RateLimit-Remaining"); + String reset = response.getFirstHeaderOrNull("RateLimit-Reset"); + + totalRequestsPerHour = limit == null ? null : Integer.valueOf(limit); + remainingRequests = remaining == null ? null : Integer.valueOf(remaining); + timeToNextAvailableRequest = reset == null ? null : millisUntilNextAvailableRequest(Long.valueOf(reset)); + } + + private static Multimap<String, String> rateLimitHeaders(HttpResponse response) { + return Multimaps.filterKeys(response.getHeaders(), new Predicate<String>() { + @Override + public boolean apply(String input) { + return input.startsWith(RATE_LIMIT_HEADER_PREFIX); + } + }); + } +} http://git-wip-us.apache.org/repos/asf/jclouds/blob/7e866ad6/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/DigitalOcean2ErrorHandler.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/DigitalOcean2ErrorHandler.java b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/DigitalOcean2ErrorHandler.java index 5eda6eb..29b7eba 100644 --- a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/DigitalOcean2ErrorHandler.java +++ b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/DigitalOcean2ErrorHandler.java @@ -20,6 +20,7 @@ import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream; import javax.inject.Singleton; +import org.jclouds.digitalocean2.exceptions.DigitalOcean2RateLimitExceededException; import org.jclouds.http.HttpCommand; import org.jclouds.http.HttpErrorHandler; import org.jclouds.http.HttpResponse; @@ -33,15 +34,16 @@ import org.jclouds.rest.ResourceNotFoundException; */ @Singleton public class DigitalOcean2ErrorHandler implements HttpErrorHandler { + public void handleError(HttpCommand command, HttpResponse response) { // it is important to always read fully and close streams byte[] data = closeClientButKeepContentStream(response); String message = data != null ? new String(data) : null; Exception exception = message != null ? new HttpResponseException(command, response, message) - : new HttpResponseException(command, response); + : new HttpResponseException(command, response); message = message != null ? message : String.format("%s -> %s", command.getCurrentRequest().getRequestLine(), - response.getStatusLine()); + response.getStatusLine()); switch (response.getStatusCode()) { case 400: break; @@ -61,6 +63,9 @@ public class DigitalOcean2ErrorHandler implements HttpErrorHandler { case 409: exception = new IllegalStateException(message, exception); break; + case 429: + exception = new DigitalOcean2RateLimitExceededException(response, exception); + break; } command.setException(exception); } http://git-wip-us.apache.org/repos/asf/jclouds/blob/7e866ad6/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandler.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandler.java b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandler.java new file mode 100644 index 0000000..d72a9fa --- /dev/null +++ b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandler.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.digitalocean2.handlers; + +import static org.jclouds.Constants.PROPERTY_MAX_RETRIES; +import static org.jclouds.digitalocean2.config.DigitalOcean2Properties.MAX_RATE_LIMIT_WAIT; + +import javax.annotation.Resource; +import javax.inject.Named; +import javax.inject.Singleton; + +import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpResponse; +import org.jclouds.http.HttpRetryHandler; +import org.jclouds.logging.Logger; + +import com.google.common.annotations.Beta; +import com.google.inject.Inject; + +/** + * Retry handler that takes into account the DigitalOcean rate limit and delays + * the requests until they are known to succeed. + */ +@Beta +@Singleton +public class RateLimitRetryHandler implements HttpRetryHandler { + + static final String RATE_LIMIT_RESET_HEADER = "RateLimit-Reset"; + + @Resource + protected Logger logger = Logger.NULL; + + @Inject(optional = true) + @Named(PROPERTY_MAX_RETRIES) + private int retryCountLimit = 5; + + @Inject(optional = true) + @Named(MAX_RATE_LIMIT_WAIT) + private int maxRateLimitWait = 120000; + + @Override + public boolean shouldRetryRequest(final HttpCommand command, final HttpResponse response) { + command.incrementFailureCount(); + + // Do not retry client errors that are not rate limit errors + if (response.getStatusCode() != 429) { + return false; + } else if (!command.isReplayable()) { + logger.error("Cannot retry after rate limit error, command is not replayable: %1$s", command); + return false; + } else if (command.getFailureCount() > retryCountLimit) { + logger.error("Cannot retry after rate limit error, command has exceeded retry limit %1$d: %2$s", + retryCountLimit, command); + return false; + } else { + return delayRequestUntilAllowed(command, response); + } + } + + private boolean delayRequestUntilAllowed(final HttpCommand command, final HttpResponse response) { + // The header is the Unix epoch time when the next request can be done + String epochForNextAvailableRequest = response.getFirstHeaderOrNull(RATE_LIMIT_RESET_HEADER); + if (epochForNextAvailableRequest == null) { + logger.error("Cannot retry after rate limit error, no retry information provided in the response"); + return false; + } + + long waitPeriod = millisUntilNextAvailableRequest(Long.parseLong(epochForNextAvailableRequest)); + + if (waitPeriod > 0) { + if (waitPeriod > maxRateLimitWait) { + logger.error("Max wait for rate limited requests is %s seconds but need to wait %s seconds, aborting", + maxRateLimitWait, waitPeriod); + return false; + } + + try { + logger.debug("Waiting %s seconds before retrying, as defined by the rate limit", waitPeriod); + // Do not use Uninterrumpibles or similar, to let the jclouds + // tiemout configuration interrupt this thread + Thread.sleep(waitPeriod); + } catch (InterruptedException ex) { + // If the request is being executed and has a timeout configured, + // the thread may be interrupted when the timeout is reached. + logger.error("Request execution was interrupted, aborting"); + Thread.currentThread().interrupt(); + return false; + } + } + + return true; + } + + public static long millisUntilNextAvailableRequest(long epochForNextAvailableRequest) { + return (epochForNextAvailableRequest * 1000) - System.currentTimeMillis(); + } +} http://git-wip-us.apache.org/repos/asf/jclouds/blob/7e866ad6/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/compute/DigitalOcean2ComputeServiceLiveTest.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/compute/DigitalOcean2ComputeServiceLiveTest.java b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/compute/DigitalOcean2ComputeServiceLiveTest.java index b8fbbe7..f45a73f 100644 --- a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/compute/DigitalOcean2ComputeServiceLiveTest.java +++ b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/compute/DigitalOcean2ComputeServiceLiveTest.java @@ -18,8 +18,10 @@ package org.jclouds.digitalocean2.compute; import org.jclouds.compute.domain.NodeMetadata; import org.jclouds.compute.internal.BaseComputeServiceLiveTest; +import org.jclouds.digitalocean2.config.DigitalOcean2RateLimitModule; import org.jclouds.sshj.config.SshjSshClientModule; import org.testng.annotations.Test; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.inject.Module; @@ -40,6 +42,12 @@ public class DigitalOcean2ComputeServiceLiveTest extends BaseComputeServiceLiveT } @Override + protected Iterable<Module> setupModules() { + return ImmutableSet.<Module> builder().addAll(super.setupModules()).add(new DigitalOcean2RateLimitModule()) + .build(); + } + + @Override public void testOptionToNotBlock() throws Exception { // DigitalOcean ComputeService implementation has to block until the node // is provisioned, to be able to return it. http://git-wip-us.apache.org/repos/asf/jclouds/blob/7e866ad6/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/exceptions/RateLimitExceptionMockTest.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/exceptions/RateLimitExceptionMockTest.java b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/exceptions/RateLimitExceptionMockTest.java new file mode 100644 index 0000000..e7831a5 --- /dev/null +++ b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/exceptions/RateLimitExceptionMockTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.digitalocean2.exceptions; + +import static org.jclouds.Constants.PROPERTY_MAX_RETRIES; +import static org.jclouds.digitalocean2.handlers.RateLimitRetryHandler.millisUntilNextAvailableRequest; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.Properties; + +import org.jclouds.digitalocean2.internal.BaseDigitalOcean2ApiMockTest; +import org.testng.annotations.Test; + +import com.squareup.okhttp.mockwebserver.MockResponse; + +@Test(groups = "unit", testName = "RateLimitExceptionMockTest", singleThreaded = true) +public class RateLimitExceptionMockTest extends BaseDigitalOcean2ApiMockTest { + + @Override + protected Properties overrides() { + Properties overrides = super.overrides(); + overrides.put(PROPERTY_MAX_RETRIES, "0"); // Do not retry + return overrides; + } + + public void testRateLimitExceptionIsThrown() throws InterruptedException { + long reset = (System.currentTimeMillis() / 1000) + 3600; // Epoch for one + // hour from now + long millisToReset = millisUntilNextAvailableRequest(reset); + + server.enqueue(new MockResponse().setResponseCode(429).addHeader("RateLimit-Limit", "5000") + .addHeader("RateLimit-Remaining", "1235").addHeader("RateLimit-Reset", String.valueOf(reset))); + + try { + api.keyApi().list(); + fail("Expected a DigitalOcean2RateLimitExceededException to be thrown"); + } catch (DigitalOcean2RateLimitExceededException ex) { + assertEquals(ex.totalRequestsPerHour().intValue(), 5000); + assertEquals(ex.remainingRequests().intValue(), 1235); + // Can't verify with millisecond precision. Use an interval to have a + // consistent test. + assertTrue(ex.timeToNextAvailableRequest() < millisToReset + && ex.timeToNextAvailableRequest() > millisToReset - 1800000); + } + } + +} http://git-wip-us.apache.org/repos/asf/jclouds/blob/7e866ad6/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandlerTest.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandlerTest.java b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandlerTest.java new file mode 100644 index 0000000..6c7c87f --- /dev/null +++ b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandlerTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.digitalocean2.handlers; + +import static org.jclouds.digitalocean2.handlers.RateLimitRetryHandler.RATE_LIMIT_RESET_HEADER; +import static org.jclouds.http.HttpUtils.releasePayload; +import static org.jclouds.io.Payloads.newInputStreamPayload; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.util.concurrent.TimeUnit; + +import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpRequest; +import org.jclouds.http.HttpResponse; +import org.jclouds.io.Payload; +import org.testng.annotations.Test; + +import com.google.common.util.concurrent.Uninterruptibles; + +@Test(groups = "unit", testName = "RateLimitRetryHandlerTest") +public class RateLimitRetryHandlerTest { + + // Configure a safe timeout of one minute to abort the tests in case they get + // stuck + private static final long TEST_SAFE_TIMEOUT = 60000; + + private final RateLimitRetryHandler rateLimitRetryHandler = new RateLimitRetryHandler(); + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDoNotRetryIfNoRateLimit() { + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(450).build(); + + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDoNotRetryIfNotReplayable() { + // InputStream payloads are not replayable + Payload payload = newInputStreamPayload(new ByteArrayInputStream(new byte[0])); + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";) + .payload(payload).build()); + HttpResponse response = HttpResponse.builder().statusCode(429).build(); + + try { + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } finally { + releasePayload(command.getCurrentRequest()); + } + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDoNotRetryIfNoRateLimitResetHeader() { + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(429).build(); + + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDoNotRetryIfTooMuchWait() { + // 5 minutes Unix epoch timestamp + long rateLimitResetEpoch = (System.currentTimeMillis() + 300000) / 1000; + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(429) + .addHeader(RATE_LIMIT_RESET_HEADER, String.valueOf(rateLimitResetEpoch)).build(); + + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testRequestIsDelayed() { + // 5 seconds Unix epoch timestamp + long rateLimitResetEpoch = (System.currentTimeMillis() + 5000) / 1000; + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(429) + .addHeader(RATE_LIMIT_RESET_HEADER, String.valueOf(rateLimitResetEpoch)).build(); + + long start = System.currentTimeMillis(); + + assertTrue(rateLimitRetryHandler.shouldRetryRequest(command, response)); + // Should have blocked the amount of time configured in the header. Use a + // smaller value to compensate the time it takes to reach the code that + // computes the amount of time to wait. + assertTrue(System.currentTimeMillis() - start > 2500); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDoNotRetryIfRequestIsAborted() throws Exception { + // 10 seconds Unix epoch timestamp + long rateLimitResetEpoch = (System.currentTimeMillis() + 10000) / 1000; + final HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";) + .build()); + final HttpResponse response = HttpResponse.builder().statusCode(429) + .addHeader(RATE_LIMIT_RESET_HEADER, String.valueOf(rateLimitResetEpoch)).build(); + + final Thread requestThread = Thread.currentThread(); + Thread killer = new Thread() { + @Override + public void run() { + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + requestThread.interrupt(); + } + }; + + // Start the killer thread that will abort the rate limit wait + killer.start(); + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testIncrementsFailureCount() { + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(429).build(); + + rateLimitRetryHandler.shouldRetryRequest(command, response); + assertEquals(command.getFailureCount(), 1); + + rateLimitRetryHandler.shouldRetryRequest(command, response); + assertEquals(command.getFailureCount(), 2); + + rateLimitRetryHandler.shouldRetryRequest(command, response); + assertEquals(command.getFailureCount(), 3); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDisallowExcessiveRetries() { + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(429).addHeader(RATE_LIMIT_RESET_HEADER, "0").build(); + + for (int i = 0; i < 5; i++) { + assertTrue(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } +} http://git-wip-us.apache.org/repos/asf/jclouds/blob/7e866ad6/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/internal/BaseDigitalOcean2ApiLiveTest.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/internal/BaseDigitalOcean2ApiLiveTest.java b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/internal/BaseDigitalOcean2ApiLiveTest.java index 18f97c6..b210c93 100644 --- a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/internal/BaseDigitalOcean2ApiLiveTest.java +++ b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/internal/BaseDigitalOcean2ApiLiveTest.java @@ -28,6 +28,7 @@ import org.jclouds.apis.BaseApiLiveTest; import org.jclouds.compute.config.ComputeServiceProperties; import org.jclouds.compute.domain.NodeMetadata; import org.jclouds.digitalocean2.DigitalOcean2Api; +import org.jclouds.digitalocean2.config.DigitalOcean2RateLimitModule; import org.jclouds.digitalocean2.domain.Action; import org.jclouds.digitalocean2.domain.Image; import org.jclouds.digitalocean2.domain.Region; @@ -35,6 +36,7 @@ import org.jclouds.digitalocean2.domain.Size; import com.google.common.base.Predicate; import com.google.common.collect.ComparisonChain; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Ordering; import com.google.inject.Injector; import com.google.inject.Key; @@ -68,6 +70,11 @@ public class BaseDigitalOcean2ApiLiveTest extends BaseApiLiveTest<DigitalOcean2A return injector.getInstance(DigitalOcean2Api.class); } + @Override protected Iterable<Module> setupModules() { + return ImmutableSet.<Module> builder().addAll(super.setupModules()).add(new DigitalOcean2RateLimitModule()) + .build(); + } + protected void assertActionCompleted(int actionId) { checkState(actionCompleted.apply(actionId), "Timeout waiting for action: %s", actionId); Action action = api.actionApi().get(actionId); http://git-wip-us.apache.org/repos/asf/jclouds/blob/7e866ad6/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/internal/BaseDigitalOcean2ApiMockTest.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/internal/BaseDigitalOcean2ApiMockTest.java b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/internal/BaseDigitalOcean2ApiMockTest.java index 78550a5..ca0c4bd 100644 --- a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/internal/BaseDigitalOcean2ApiMockTest.java +++ b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/internal/BaseDigitalOcean2ApiMockTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertEquals; import java.io.IOException; import java.util.Map; +import java.util.Properties; import java.util.Set; import org.jclouds.ContextBuilder; @@ -59,7 +60,6 @@ public class BaseDigitalOcean2ApiMockTest { // So that we can ignore formatting. private final JsonParser parser = new JsonParser(); - @BeforeMethod public void start() throws IOException { server = new MockWebServer(); @@ -68,6 +68,7 @@ public class BaseDigitalOcean2ApiMockTest { .credentials("", MOCK_BEARER_TOKEN) .endpoint(url("")) .modules(modules) + .overrides(overrides()) .build(); json = ctx.utils().injector().getInstance(Json.class); api = ctx.getApi(); @@ -79,6 +80,10 @@ public class BaseDigitalOcean2ApiMockTest { api.close(); } + protected Properties overrides() { + return new Properties(); + } + protected String url(String path) { return server.getUrl(path).toString(); }
