Repository: jclouds Updated Branches: refs/heads/master edbb2c0e6 -> 8030e53f3
Move digital ocean rate limit handler to core to make it reusable Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/8030e53f Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/8030e53f Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/8030e53f Branch: refs/heads/master Commit: 8030e53f3236a1a09c76b60a0694e10b0456575e Parents: edbb2c0 Author: Ignasi Barrera <[email protected]> Authored: Wed Oct 19 16:56:03 2016 +0200 Committer: Ignasi Barrera <[email protected]> Committed: Wed Oct 19 16:56:51 2016 +0200 ---------------------------------------------------------------------- core/src/main/java/org/jclouds/Constants.java | 8 + .../http/handlers/RateLimitRetryHandler.java | 130 ++++++++++++++++ .../handlers/RateLimitRetryHandlerTest.java | 152 ++++++++++++++++++ .../config/DigitalOcean2Properties.java | 33 ---- .../config/DigitalOcean2RateLimitModule.java | 4 +- ...DigitalOcean2RateLimitExceededException.java | 2 +- .../DigitalOcean2RateLimitRetryHandler.java | 45 ++++++ .../handlers/RateLimitRetryHandler.java | 111 -------------- .../exceptions/RateLimitExceptionMockTest.java | 2 +- .../handlers/RateLimitRetryHandlerTest.java | 153 ------------------- 10 files changed, 339 insertions(+), 301 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds/blob/8030e53f/core/src/main/java/org/jclouds/Constants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/jclouds/Constants.java b/core/src/main/java/org/jclouds/Constants.java index 5c87ee6..f9d89f8 100644 --- a/core/src/main/java/org/jclouds/Constants.java +++ b/core/src/main/java/org/jclouds/Constants.java @@ -352,6 +352,14 @@ public final class Constants { /** Comma-separated list of methods considered idempotent for purposes of retries. By default jclouds uses DELETE,GET,HEAD,OPTIONS,PUT. */ public static final String PROPERTY_IDEMPOTENT_METHODS = "jclouds.idempotent-methods"; + + /** + * Maximum amount of time (in milliseconds) a request will wait until retrying if + * the rate limit is exhausted. + * <p> + * Default value: 2 minutes. + */ + public static final String PROPERTY_MAX_RATE_LIMIT_WAIT = "jclouds.max-ratelimit-wait"; private Constants() { throw new AssertionError("intentionally unimplemented"); http://git-wip-us.apache.org/repos/asf/jclouds/blob/8030e53f/core/src/main/java/org/jclouds/http/handlers/RateLimitRetryHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/jclouds/http/handlers/RateLimitRetryHandler.java b/core/src/main/java/org/jclouds/http/handlers/RateLimitRetryHandler.java new file mode 100644 index 0000000..862ffd4 --- /dev/null +++ b/core/src/main/java/org/jclouds/http/handlers/RateLimitRetryHandler.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.http.handlers; + +import static org.jclouds.Constants.PROPERTY_MAX_RATE_LIMIT_WAIT; +import static org.jclouds.Constants.PROPERTY_MAX_RETRIES; + +import javax.annotation.Resource; +import javax.inject.Named; + +import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpResponse; +import org.jclouds.http.HttpRetryHandler; +import org.jclouds.logging.Logger; + +import com.google.common.annotations.Beta; +import com.google.common.base.Optional; +import com.google.inject.Inject; + +/** + * Retry handler that takes into account the provider rate limit and delays the + * requests until they are known to succeed. + */ +@Beta +public abstract class RateLimitRetryHandler implements HttpRetryHandler { + + @Resource + protected Logger logger = Logger.NULL; + + @Inject(optional = true) + @Named(PROPERTY_MAX_RETRIES) + private int retryCountLimit = 5; + + @Inject(optional = true) + @Named(PROPERTY_MAX_RATE_LIMIT_WAIT) + private int maxRateLimitWait = 2 * 60 * 1000; + + /** + * Returns the response status that will be considered a rate limit error. + * <p> + * Providers can override this to customize which responses are retried. + */ + protected int rateLimitErrorStatus() { + return 429; + } + + /** + * Compute the number of milliseconds that must pass until a request can be + * performed. + * + * @param command The command being executed. + * @param response The rate-limit error response. + * @return The number of milliseconds to wait for an available request, if taht information is available. + */ + protected abstract Optional<Long> millisToNextAvailableRequest(final HttpCommand command, final HttpResponse response); + + @Override + public boolean shouldRetryRequest(final HttpCommand command, final HttpResponse response) { + command.incrementFailureCount(); + + // Do not retry client errors that are not rate limit errors + if (response.getStatusCode() != rateLimitErrorStatus()) { + return false; + } else if (!command.isReplayable()) { + logger.error("Cannot retry after rate limit error, command is not replayable: %1$s", command); + return false; + } else if (command.getFailureCount() > retryCountLimit) { + logger.error("Cannot retry after rate limit error, command has exceeded retry limit %1$d: %2$s", + retryCountLimit, command); + return false; + } else { + return delayRequestUntilAllowed(command, response); + } + } + + private boolean delayRequestUntilAllowed(final HttpCommand command, final HttpResponse response) { + Optional<Long> millisToNextAvailableRequest = millisToNextAvailableRequest(command, response); + if (!millisToNextAvailableRequest.isPresent()) { + logger.error("Cannot retry after rate limit error, no retry information provided in the response"); + return false; + } + + long waitPeriod = millisToNextAvailableRequest.get(); + if (waitPeriod > 0L) { + if (waitPeriod > maxRateLimitWait) { + logger.error("Max wait for rate limited requests is %sms but need to wait %sms, aborting", + maxRateLimitWait, waitPeriod); + return false; + } + + try { + logger.debug("Waiting %sms before retrying, as defined by the rate limit", waitPeriod); + // Do not use Uninterrumpibles or similar, to let the jclouds + // tiemout configuration interrupt this thread + Thread.sleep(waitPeriod); + } catch (InterruptedException ex) { + // If the request is being executed and has a timeout configured, + // the thread may be interrupted when the timeout is reached. + logger.error("Request execution was interrupted, aborting"); + Thread.currentThread().interrupt(); + return false; + } + } + + return true; + } + + public int getRetryCountLimit() { + return retryCountLimit; + } + + public int getMaxRateLimitWait() { + return maxRateLimitWait; + } + +} http://git-wip-us.apache.org/repos/asf/jclouds/blob/8030e53f/core/src/test/java/org/jclouds/http/handlers/RateLimitRetryHandlerTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/jclouds/http/handlers/RateLimitRetryHandlerTest.java b/core/src/test/java/org/jclouds/http/handlers/RateLimitRetryHandlerTest.java new file mode 100644 index 0000000..a472330 --- /dev/null +++ b/core/src/test/java/org/jclouds/http/handlers/RateLimitRetryHandlerTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.http.handlers; + +import static com.google.common.net.HttpHeaders.RETRY_AFTER; +import static org.jclouds.http.HttpUtils.releasePayload; +import static org.jclouds.io.Payloads.newInputStreamPayload; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.util.concurrent.TimeUnit; + +import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpRequest; +import org.jclouds.http.HttpResponse; +import org.jclouds.io.Payload; +import org.testng.annotations.Test; + +import com.google.common.base.Optional; +import com.google.common.util.concurrent.Uninterruptibles; + +@Test(groups = "unit", testName = "RateLimitRetryHandlerTest") +public class RateLimitRetryHandlerTest { + + // Configure a safe timeout of one minute to abort the tests in case they get + // stuck + private static final long TEST_SAFE_TIMEOUT = 60000; + + private final RateLimitRetryHandler rateLimitRetryHandler = new RateLimitRetryHandler() { + @Override + protected Optional<Long> millisToNextAvailableRequest(HttpCommand command, HttpResponse response) { + String secondsToNextAvailableRequest = response.getFirstHeaderOrNull(RETRY_AFTER); + return secondsToNextAvailableRequest != null ? Optional.of(Long.valueOf(secondsToNextAvailableRequest) * 1000) + : Optional.<Long> absent(); + } + }; + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDoNotRetryIfNoRateLimit() { + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(450).build(); + + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDoNotRetryIfNotReplayable() { + // InputStream payloads are not replayable + Payload payload = newInputStreamPayload(new ByteArrayInputStream(new byte[0])); + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";) + .payload(payload).build()); + HttpResponse response = HttpResponse.builder().statusCode(429).build(); + + try { + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } finally { + releasePayload(command.getCurrentRequest()); + } + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDoNotRetryIfNoRateLimitInfo() { + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(429).build(); + + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDoNotRetryIfTooMuchWait() { + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(429).addHeader(RETRY_AFTER, "400").build(); + + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testRequestIsDelayed() { + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(429).addHeader(RETRY_AFTER, "5").build(); + + long start = System.currentTimeMillis(); + + assertTrue(rateLimitRetryHandler.shouldRetryRequest(command, response)); + // Should have blocked the amount of time configured in the header. Use a + // smaller value to compensate the time it takes to reach the code that + // computes the amount of time to wait. + assertTrue(System.currentTimeMillis() - start > 2500); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDoNotRetryIfRequestIsAborted() throws Exception { + final HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";) + .build()); + final HttpResponse response = HttpResponse.builder().statusCode(429).addHeader(RETRY_AFTER, "10").build(); + + final Thread requestThread = Thread.currentThread(); + Thread killer = new Thread() { + @Override + public void run() { + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + requestThread.interrupt(); + } + }; + + // Start the killer thread that will abort the rate limit wait + killer.start(); + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testIncrementsFailureCount() { + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(429).build(); + + rateLimitRetryHandler.shouldRetryRequest(command, response); + assertEquals(command.getFailureCount(), 1); + + rateLimitRetryHandler.shouldRetryRequest(command, response); + assertEquals(command.getFailureCount(), 2); + + rateLimitRetryHandler.shouldRetryRequest(command, response); + assertEquals(command.getFailureCount(), 3); + } + + @Test(timeOut = TEST_SAFE_TIMEOUT) + public void testDisallowExcessiveRetries() { + HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); + HttpResponse response = HttpResponse.builder().statusCode(429).addHeader(RETRY_AFTER, "0").build(); + + for (int i = 0; i < 5; i++) { + assertTrue(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } + assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); + } +} http://git-wip-us.apache.org/repos/asf/jclouds/blob/8030e53f/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2Properties.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2Properties.java b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2Properties.java deleted file mode 100644 index d0d1098..0000000 --- a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2Properties.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jclouds.digitalocean2.config; - -public final class DigitalOcean2Properties { - - /** - * Maximum amount of time (in milliseconds) a request will wait until retrying if - * the rate limit is exhausted. - * <p> - * Default value: 2 minutes. - */ - public static final String MAX_RATE_LIMIT_WAIT = "jclouds.max-ratelimit-wait"; - - private DigitalOcean2Properties() { - throw new AssertionError("intentionally unimplemented"); - } - -} http://git-wip-us.apache.org/repos/asf/jclouds/blob/8030e53f/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2RateLimitModule.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2RateLimitModule.java b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2RateLimitModule.java index 1b0a95f..3d8f1e3 100644 --- a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2RateLimitModule.java +++ b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/config/DigitalOcean2RateLimitModule.java @@ -16,7 +16,7 @@ */ package org.jclouds.digitalocean2.config; -import org.jclouds.digitalocean2.handlers.RateLimitRetryHandler; +import org.jclouds.digitalocean2.handlers.DigitalOcean2RateLimitRetryHandler; import org.jclouds.http.HttpRetryHandler; import org.jclouds.http.annotation.ClientError; @@ -25,6 +25,6 @@ import com.google.inject.AbstractModule; public class DigitalOcean2RateLimitModule extends AbstractModule { @Override protected void configure() { - bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(RateLimitRetryHandler.class); + bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(DigitalOcean2RateLimitRetryHandler.class); } } http://git-wip-us.apache.org/repos/asf/jclouds/blob/8030e53f/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/exceptions/DigitalOcean2RateLimitExceededException.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/exceptions/DigitalOcean2RateLimitExceededException.java b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/exceptions/DigitalOcean2RateLimitExceededException.java index 8218a26..3bf76c3 100644 --- a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/exceptions/DigitalOcean2RateLimitExceededException.java +++ b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/exceptions/DigitalOcean2RateLimitExceededException.java @@ -16,7 +16,7 @@ */ package org.jclouds.digitalocean2.exceptions; -import static org.jclouds.digitalocean2.handlers.RateLimitRetryHandler.millisUntilNextAvailableRequest; +import static org.jclouds.digitalocean2.handlers.DigitalOcean2RateLimitRetryHandler.millisUntilNextAvailableRequest; import org.jclouds.http.HttpResponse; import org.jclouds.rest.RateLimitExceededException; http://git-wip-us.apache.org/repos/asf/jclouds/blob/8030e53f/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/DigitalOcean2RateLimitRetryHandler.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/DigitalOcean2RateLimitRetryHandler.java b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/DigitalOcean2RateLimitRetryHandler.java new file mode 100644 index 0000000..debb8f8 --- /dev/null +++ b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/DigitalOcean2RateLimitRetryHandler.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.digitalocean2.handlers; + +import javax.inject.Singleton; + +import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpResponse; +import org.jclouds.http.handlers.RateLimitRetryHandler; + +import com.google.common.base.Optional; + +@Singleton +public class DigitalOcean2RateLimitRetryHandler extends RateLimitRetryHandler { + + @Override + protected Optional<Long> millisToNextAvailableRequest(HttpCommand command, HttpResponse response) { + // The header is the Unix epoch time when the next request can be done + String epochForNextAvailableRequest = response.getFirstHeaderOrNull("RateLimit-Reset"); + if (epochForNextAvailableRequest == null) { + return Optional.absent(); + } + return Optional.of(millisUntilNextAvailableRequest(Long.parseLong(epochForNextAvailableRequest))); + } + + public static long millisUntilNextAvailableRequest(long epochForNextAvailableRequest) { + return (epochForNextAvailableRequest * 1000) - System.currentTimeMillis(); + } + + +} http://git-wip-us.apache.org/repos/asf/jclouds/blob/8030e53f/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandler.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandler.java b/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandler.java deleted file mode 100644 index d72a9fa..0000000 --- a/providers/digitalocean2/src/main/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandler.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jclouds.digitalocean2.handlers; - -import static org.jclouds.Constants.PROPERTY_MAX_RETRIES; -import static org.jclouds.digitalocean2.config.DigitalOcean2Properties.MAX_RATE_LIMIT_WAIT; - -import javax.annotation.Resource; -import javax.inject.Named; -import javax.inject.Singleton; - -import org.jclouds.http.HttpCommand; -import org.jclouds.http.HttpResponse; -import org.jclouds.http.HttpRetryHandler; -import org.jclouds.logging.Logger; - -import com.google.common.annotations.Beta; -import com.google.inject.Inject; - -/** - * Retry handler that takes into account the DigitalOcean rate limit and delays - * the requests until they are known to succeed. - */ -@Beta -@Singleton -public class RateLimitRetryHandler implements HttpRetryHandler { - - static final String RATE_LIMIT_RESET_HEADER = "RateLimit-Reset"; - - @Resource - protected Logger logger = Logger.NULL; - - @Inject(optional = true) - @Named(PROPERTY_MAX_RETRIES) - private int retryCountLimit = 5; - - @Inject(optional = true) - @Named(MAX_RATE_LIMIT_WAIT) - private int maxRateLimitWait = 120000; - - @Override - public boolean shouldRetryRequest(final HttpCommand command, final HttpResponse response) { - command.incrementFailureCount(); - - // Do not retry client errors that are not rate limit errors - if (response.getStatusCode() != 429) { - return false; - } else if (!command.isReplayable()) { - logger.error("Cannot retry after rate limit error, command is not replayable: %1$s", command); - return false; - } else if (command.getFailureCount() > retryCountLimit) { - logger.error("Cannot retry after rate limit error, command has exceeded retry limit %1$d: %2$s", - retryCountLimit, command); - return false; - } else { - return delayRequestUntilAllowed(command, response); - } - } - - private boolean delayRequestUntilAllowed(final HttpCommand command, final HttpResponse response) { - // The header is the Unix epoch time when the next request can be done - String epochForNextAvailableRequest = response.getFirstHeaderOrNull(RATE_LIMIT_RESET_HEADER); - if (epochForNextAvailableRequest == null) { - logger.error("Cannot retry after rate limit error, no retry information provided in the response"); - return false; - } - - long waitPeriod = millisUntilNextAvailableRequest(Long.parseLong(epochForNextAvailableRequest)); - - if (waitPeriod > 0) { - if (waitPeriod > maxRateLimitWait) { - logger.error("Max wait for rate limited requests is %s seconds but need to wait %s seconds, aborting", - maxRateLimitWait, waitPeriod); - return false; - } - - try { - logger.debug("Waiting %s seconds before retrying, as defined by the rate limit", waitPeriod); - // Do not use Uninterrumpibles or similar, to let the jclouds - // tiemout configuration interrupt this thread - Thread.sleep(waitPeriod); - } catch (InterruptedException ex) { - // If the request is being executed and has a timeout configured, - // the thread may be interrupted when the timeout is reached. - logger.error("Request execution was interrupted, aborting"); - Thread.currentThread().interrupt(); - return false; - } - } - - return true; - } - - public static long millisUntilNextAvailableRequest(long epochForNextAvailableRequest) { - return (epochForNextAvailableRequest * 1000) - System.currentTimeMillis(); - } -} http://git-wip-us.apache.org/repos/asf/jclouds/blob/8030e53f/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/exceptions/RateLimitExceptionMockTest.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/exceptions/RateLimitExceptionMockTest.java b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/exceptions/RateLimitExceptionMockTest.java index e7831a5..6b90626 100644 --- a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/exceptions/RateLimitExceptionMockTest.java +++ b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/exceptions/RateLimitExceptionMockTest.java @@ -17,7 +17,7 @@ package org.jclouds.digitalocean2.exceptions; import static org.jclouds.Constants.PROPERTY_MAX_RETRIES; -import static org.jclouds.digitalocean2.handlers.RateLimitRetryHandler.millisUntilNextAvailableRequest; +import static org.jclouds.digitalocean2.handlers.DigitalOcean2RateLimitRetryHandler.millisUntilNextAvailableRequest; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; http://git-wip-us.apache.org/repos/asf/jclouds/blob/8030e53f/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandlerTest.java ---------------------------------------------------------------------- diff --git a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandlerTest.java b/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandlerTest.java deleted file mode 100644 index 6c7c87f..0000000 --- a/providers/digitalocean2/src/test/java/org/jclouds/digitalocean2/handlers/RateLimitRetryHandlerTest.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jclouds.digitalocean2.handlers; - -import static org.jclouds.digitalocean2.handlers.RateLimitRetryHandler.RATE_LIMIT_RESET_HEADER; -import static org.jclouds.http.HttpUtils.releasePayload; -import static org.jclouds.io.Payloads.newInputStreamPayload; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - -import java.io.ByteArrayInputStream; -import java.util.concurrent.TimeUnit; - -import org.jclouds.http.HttpCommand; -import org.jclouds.http.HttpRequest; -import org.jclouds.http.HttpResponse; -import org.jclouds.io.Payload; -import org.testng.annotations.Test; - -import com.google.common.util.concurrent.Uninterruptibles; - -@Test(groups = "unit", testName = "RateLimitRetryHandlerTest") -public class RateLimitRetryHandlerTest { - - // Configure a safe timeout of one minute to abort the tests in case they get - // stuck - private static final long TEST_SAFE_TIMEOUT = 60000; - - private final RateLimitRetryHandler rateLimitRetryHandler = new RateLimitRetryHandler(); - - @Test(timeOut = TEST_SAFE_TIMEOUT) - public void testDoNotRetryIfNoRateLimit() { - HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); - HttpResponse response = HttpResponse.builder().statusCode(450).build(); - - assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); - } - - @Test(timeOut = TEST_SAFE_TIMEOUT) - public void testDoNotRetryIfNotReplayable() { - // InputStream payloads are not replayable - Payload payload = newInputStreamPayload(new ByteArrayInputStream(new byte[0])); - HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";) - .payload(payload).build()); - HttpResponse response = HttpResponse.builder().statusCode(429).build(); - - try { - assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); - } finally { - releasePayload(command.getCurrentRequest()); - } - } - - @Test(timeOut = TEST_SAFE_TIMEOUT) - public void testDoNotRetryIfNoRateLimitResetHeader() { - HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); - HttpResponse response = HttpResponse.builder().statusCode(429).build(); - - assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); - } - - @Test(timeOut = TEST_SAFE_TIMEOUT) - public void testDoNotRetryIfTooMuchWait() { - // 5 minutes Unix epoch timestamp - long rateLimitResetEpoch = (System.currentTimeMillis() + 300000) / 1000; - HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); - HttpResponse response = HttpResponse.builder().statusCode(429) - .addHeader(RATE_LIMIT_RESET_HEADER, String.valueOf(rateLimitResetEpoch)).build(); - - assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); - } - - @Test(timeOut = TEST_SAFE_TIMEOUT) - public void testRequestIsDelayed() { - // 5 seconds Unix epoch timestamp - long rateLimitResetEpoch = (System.currentTimeMillis() + 5000) / 1000; - HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); - HttpResponse response = HttpResponse.builder().statusCode(429) - .addHeader(RATE_LIMIT_RESET_HEADER, String.valueOf(rateLimitResetEpoch)).build(); - - long start = System.currentTimeMillis(); - - assertTrue(rateLimitRetryHandler.shouldRetryRequest(command, response)); - // Should have blocked the amount of time configured in the header. Use a - // smaller value to compensate the time it takes to reach the code that - // computes the amount of time to wait. - assertTrue(System.currentTimeMillis() - start > 2500); - } - - @Test(timeOut = TEST_SAFE_TIMEOUT) - public void testDoNotRetryIfRequestIsAborted() throws Exception { - // 10 seconds Unix epoch timestamp - long rateLimitResetEpoch = (System.currentTimeMillis() + 10000) / 1000; - final HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";) - .build()); - final HttpResponse response = HttpResponse.builder().statusCode(429) - .addHeader(RATE_LIMIT_RESET_HEADER, String.valueOf(rateLimitResetEpoch)).build(); - - final Thread requestThread = Thread.currentThread(); - Thread killer = new Thread() { - @Override - public void run() { - Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); - requestThread.interrupt(); - } - }; - - // Start the killer thread that will abort the rate limit wait - killer.start(); - assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); - } - - @Test(timeOut = TEST_SAFE_TIMEOUT) - public void testIncrementsFailureCount() { - HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); - HttpResponse response = HttpResponse.builder().statusCode(429).build(); - - rateLimitRetryHandler.shouldRetryRequest(command, response); - assertEquals(command.getFailureCount(), 1); - - rateLimitRetryHandler.shouldRetryRequest(command, response); - assertEquals(command.getFailureCount(), 2); - - rateLimitRetryHandler.shouldRetryRequest(command, response); - assertEquals(command.getFailureCount(), 3); - } - - @Test(timeOut = TEST_SAFE_TIMEOUT) - public void testDisallowExcessiveRetries() { - HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost";).build()); - HttpResponse response = HttpResponse.builder().statusCode(429).addHeader(RATE_LIMIT_RESET_HEADER, "0").build(); - - for (int i = 0; i < 5; i++) { - assertTrue(rateLimitRetryHandler.shouldRetryRequest(command, response)); - } - assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); - } -}
