Repository: jclouds Updated Branches: refs/heads/1.7.x a38cbd580 -> fe7988566
JCLOUDS-460: Add jitter to avoid thundering herd When issuing many simultaneous requests to Synaptic Atmos I observed: HTTP/1.1 failed with code 500, error: AtmosError [code=1040, message=The server is busy. Please try again.] Previously all clients slept for fixed intervals and thus retried around the same time. This commit adds a random delay which should better distribute load on the provider. Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/fe798856 Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/fe798856 Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/fe798856 Branch: refs/heads/1.7.x Commit: fe79885663bc2c6d5884be94db72b55268f7fd0c Parents: a38cbd5 Author: Andrew Gaul <[email protected]> Authored: Thu Apr 18 12:54:00 2013 -0700 Committer: Andrew Gaul <[email protected]> Committed: Mon Mar 3 22:30:50 2014 -0800 ---------------------------------------------------------------------- .../org/jclouds/http/handlers/BackoffLimitedRetryHandler.java | 4 ++++ 1 file changed, 4 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds/blob/fe798856/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java b/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java index 5162042..5d8b5b1 100644 --- a/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java +++ b/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java @@ -19,6 +19,7 @@ package org.jclouds.http.handlers; import static org.jclouds.http.HttpUtils.releasePayload; import java.io.IOException; +import java.util.Random; import javax.annotation.Resource; import javax.inject.Named; @@ -126,6 +127,9 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException public void imposeBackoffExponentialDelay(long period, long maxPeriod, int pow, int failureCount, int max, String commandDescription) { long delayMs = (long) (period * Math.pow(failureCount, pow)); + // Add random delay to avoid thundering herd problem when multiple + // simultaneous failed requests retry after sleeping for the same delay. + delayMs += new Random().nextInt((int) (delayMs / 10)); delayMs = delayMs > maxPeriod ? maxPeriod : delayMs; logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, max, delayMs, commandDescription); try {
