Updated Branches: refs/heads/master ff3405c96 -> 75385ee57
JCLOUDS-100. Add AWSServerErrorRetryHandler, with an increased back off time. Project: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/commit/75385ee5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/tree/75385ee5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/diff/75385ee5 Branch: refs/heads/master Commit: 75385ee57d93fceeef35273ddc709567af48d96c Parents: ff3405c Author: Andrew Bayer <[email protected]> Authored: Mon Jun 17 12:33:41 2013 -0700 Committer: Andrew Bayer <[email protected]> Committed: Mon Jun 17 16:48:15 2013 -0700 ---------------------------------------------------------------------- .../jclouds/s3/config/S3RestClientModule.java | 2 + .../jclouds/sqs/config/SQSRestClientModule.java | 2 + .../jclouds/aws/config/AWSHttpApiModule.java | 9 ++ .../jclouds/aws/config/AWSRestClientModule.java | 9 ++ .../handlers/AWSServerErrorRetryHandler.java | 78 +++++++++++++ .../AWSServerErrorRetryHandlerTest.java | 116 +++++++++++++++++++ 6 files changed, 216 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/75385ee5/apis/s3/src/main/java/org/jclouds/s3/config/S3RestClientModule.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/config/S3RestClientModule.java b/apis/s3/src/main/java/org/jclouds/s3/config/S3RestClientModule.java index 52a573d..33ac2ef 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/config/S3RestClientModule.java +++ b/apis/s3/src/main/java/org/jclouds/s3/config/S3RestClientModule.java @@ -27,6 +27,7 @@ import javax.inject.Singleton; import org.jclouds.Constants; import org.jclouds.aws.config.AWSRestClientModule; import org.jclouds.aws.handlers.AWSClientErrorRetryHandler; +import org.jclouds.aws.handlers.AWSServerErrorRetryHandler; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; @@ -185,6 +186,7 @@ public class S3RestClientModule<S extends S3Client, A extends S3AsyncClient> ext protected void bindRetryHandlers() { bind(HttpRetryHandler.class).annotatedWith(Redirection.class).to(S3RedirectionRetryHandler.class); bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(AWSClientErrorRetryHandler.class); + bind(HttpRetryHandler.class).annotatedWith(ServerError.class).to(AWSServerErrorRetryHandler.class); } @Provides http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/75385ee5/apis/sqs/src/main/java/org/jclouds/sqs/config/SQSRestClientModule.java ---------------------------------------------------------------------- diff --git a/apis/sqs/src/main/java/org/jclouds/sqs/config/SQSRestClientModule.java b/apis/sqs/src/main/java/org/jclouds/sqs/config/SQSRestClientModule.java index 9589ff2..69e16d5 100644 --- a/apis/sqs/src/main/java/org/jclouds/sqs/config/SQSRestClientModule.java +++ b/apis/sqs/src/main/java/org/jclouds/sqs/config/SQSRestClientModule.java @@ -22,6 +22,7 @@ import static org.jclouds.reflect.Reflection2.typeToken; import java.util.Map; import org.jclouds.aws.config.FormSigningRestClientModule; +import org.jclouds.aws.handlers.AWSServerErrorRetryHandler; import org.jclouds.http.HttpErrorHandler; import org.jclouds.http.HttpRetryHandler; import org.jclouds.http.annotation.ClientError; @@ -68,6 +69,7 @@ public class SQSRestClientModule extends FormSigningRestClientModule<SQSApi, SQS @Override protected void bindRetryHandlers() { bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(SQSErrorRetryHandler.class); + bind(HttpRetryHandler.class).annotatedWith(ServerError.class).to(AWSServerErrorRetryHandler.class); } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/75385ee5/apis/sts/src/main/java/org/jclouds/aws/config/AWSHttpApiModule.java ---------------------------------------------------------------------- diff --git a/apis/sts/src/main/java/org/jclouds/aws/config/AWSHttpApiModule.java b/apis/sts/src/main/java/org/jclouds/aws/config/AWSHttpApiModule.java index 1879aae..a509461 100644 --- a/apis/sts/src/main/java/org/jclouds/aws/config/AWSHttpApiModule.java +++ b/apis/sts/src/main/java/org/jclouds/aws/config/AWSHttpApiModule.java @@ -22,6 +22,7 @@ import java.util.Set; import javax.inject.Singleton; import org.jclouds.aws.handlers.AWSClientErrorRetryHandler; +import org.jclouds.aws.handlers.AWSServerErrorRetryHandler; import org.jclouds.aws.handlers.ParseAWSErrorFromXmlContent; import org.jclouds.http.HttpErrorHandler; import org.jclouds.http.HttpRetryHandler; @@ -56,6 +57,13 @@ public abstract class AWSHttpApiModule<A> extends HttpApiModule<A> { return ImmutableSet.of("RequestTimeout", "OperationAborted", "SignatureDoesNotMatch"); } + @Provides + @ServerError + @Singleton + protected Set<String> provideRetryableServerCodes(){ + return ImmutableSet.of("RequestLimitExceeded"); + } + @Override protected void bindErrorHandlers() { bind(HttpErrorHandler.class).annotatedWith(Redirection.class).to(ParseAWSErrorFromXmlContent.class); @@ -66,6 +74,7 @@ public abstract class AWSHttpApiModule<A> extends HttpApiModule<A> { @Override protected void bindRetryHandlers() { bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(AWSClientErrorRetryHandler.class); + bind(HttpRetryHandler.class).annotatedWith(ServerError.class).to(AWSServerErrorRetryHandler.class); } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/75385ee5/apis/sts/src/main/java/org/jclouds/aws/config/AWSRestClientModule.java ---------------------------------------------------------------------- diff --git a/apis/sts/src/main/java/org/jclouds/aws/config/AWSRestClientModule.java b/apis/sts/src/main/java/org/jclouds/aws/config/AWSRestClientModule.java index 487742b..05358a4 100644 --- a/apis/sts/src/main/java/org/jclouds/aws/config/AWSRestClientModule.java +++ b/apis/sts/src/main/java/org/jclouds/aws/config/AWSRestClientModule.java @@ -23,6 +23,7 @@ import java.util.Set; import javax.inject.Singleton; import org.jclouds.aws.handlers.AWSClientErrorRetryHandler; +import org.jclouds.aws.handlers.AWSServerErrorRetryHandler; import org.jclouds.aws.handlers.ParseAWSErrorFromXmlContent; import org.jclouds.http.HttpErrorHandler; import org.jclouds.http.HttpRetryHandler; @@ -71,6 +72,13 @@ public abstract class AWSRestClientModule<S, A> extends RestClientModule<S, A> { return ImmutableSet.of("RequestTimeout", "OperationAborted", "SignatureDoesNotMatch"); } + @Provides + @ServerError + @Singleton + protected Set<String> provideRetryableServerCodes(){ + return ImmutableSet.of("RequestLimitExceeded"); + } + @Override protected void bindErrorHandlers() { bind(HttpErrorHandler.class).annotatedWith(Redirection.class).to(ParseAWSErrorFromXmlContent.class); @@ -81,6 +89,7 @@ public abstract class AWSRestClientModule<S, A> extends RestClientModule<S, A> { @Override protected void bindRetryHandlers() { bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(AWSClientErrorRetryHandler.class); + bind(HttpRetryHandler.class).annotatedWith(ServerError.class).to(AWSServerErrorRetryHandler.class); } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/75385ee5/apis/sts/src/main/java/org/jclouds/aws/handlers/AWSServerErrorRetryHandler.java ---------------------------------------------------------------------- diff --git a/apis/sts/src/main/java/org/jclouds/aws/handlers/AWSServerErrorRetryHandler.java b/apis/sts/src/main/java/org/jclouds/aws/handlers/AWSServerErrorRetryHandler.java new file mode 100644 index 0000000..042d825 --- /dev/null +++ b/apis/sts/src/main/java/org/jclouds/aws/handlers/AWSServerErrorRetryHandler.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.aws.handlers; + +import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream; + +import java.util.Set; + +import org.jclouds.aws.domain.AWSError; +import org.jclouds.aws.util.AWSUtils; +import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpResponse; +import org.jclouds.http.HttpRetryHandler; +import org.jclouds.http.annotation.ServerError; +import org.jclouds.http.handlers.BackoffLimitedRetryHandler; + +import com.google.inject.Inject; +import com.google.inject.Singleton; + +/** + * Handles Retryable responses with error codes in the 5xx range + * + * @author Andrew Bayer + */ +@Singleton +public class AWSServerErrorRetryHandler extends BackoffLimitedRetryHandler { + + private final AWSUtils utils; + private final Set<String> retryableServerCodes; + + @Inject + public AWSServerErrorRetryHandler(AWSUtils utils, + @ServerError Set<String> retryableServerCodes) { + this.utils = utils; + this.retryableServerCodes = retryableServerCodes; + } + + @Override + public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) { + if (response.getStatusCode() == 503) { + // Content can be null in the case of HEAD requests + if (response.getPayload() != null) { + closeClientButKeepContentStream(response); + AWSError error = utils.parseAWSErrorFromContent(command.getCurrentRequest(), response); + if (error != null) { + return shouldRetryRequestOnError(command, response, error); + } + } + } + return false; + } + + protected boolean shouldRetryRequestOnError(HttpCommand command, HttpResponse response, AWSError error) { + if (retryableServerCodes.contains(error.getCode())) + return super.shouldRetryRequest(command, response); + return false; + } + + public void imposeBackoffExponentialDelay(long period, int pow, int failureCount, int max, String commandDescription) { + imposeBackoffExponentialDelay(period, period * 100l, pow, failureCount, max, commandDescription); + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/75385ee5/apis/sts/src/test/java/org/jclouds/aws/handlers/AWSServerErrorRetryHandlerTest.java ---------------------------------------------------------------------- diff --git a/apis/sts/src/test/java/org/jclouds/aws/handlers/AWSServerErrorRetryHandlerTest.java b/apis/sts/src/test/java/org/jclouds/aws/handlers/AWSServerErrorRetryHandlerTest.java new file mode 100644 index 0000000..af1a41a --- /dev/null +++ b/apis/sts/src/test/java/org/jclouds/aws/handlers/AWSServerErrorRetryHandlerTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.aws.handlers; +import static javax.ws.rs.HttpMethod.PUT; +import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; +import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.testng.Assert.assertFalse; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.easymock.IAnswer; + +import org.jclouds.aws.domain.AWSError; +import org.jclouds.aws.util.AWSUtils; +import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpRequest; +import org.jclouds.http.HttpResponse; +import org.jclouds.http.handlers.BackoffLimitedRetryHandler; +import org.jclouds.io.Payloads; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableSet; + +/** + * Tests behavior of {@code AWSServerErrorRetryHandler} + * + * @author Andrew Bayer + */ +@Test(groups = "unit", testName = "AWSServerErrorRetryHandlerTest") +public class AWSServerErrorRetryHandlerTest { + @Test + public void test500DoesNotRetry() { + + AWSUtils utils = createMock(AWSUtils.class); + HttpCommand command = createMock(HttpCommand.class); + + replay(utils, command); + + AWSServerErrorRetryHandler retry = new AWSServerErrorRetryHandler(utils, + ImmutableSet.<String> of()); + + assertFalse(retry.shouldRetryRequest(command, HttpResponse.builder().statusCode(INTERNAL_SERVER_ERROR.getStatusCode()).build())); + + verify(utils, command); + + } + + @DataProvider(name = "codes") + public Object[][] createData() { + return new Object[][] { { "RequestLimitExceeded" } }; + } + + @Test(dataProvider = "codes") + public void test503DoesBackoffAndRetryForCode(String code) { + + AWSUtils utils = createMock(AWSUtils.class); + HttpCommand command = createMock(HttpCommand.class); + + HttpRequest putBucket = HttpRequest.builder().method(PUT) + .endpoint("https://adriancole-blobstore113.s3.amazonaws.com/";).build(); + + HttpResponse limitExceeded = HttpResponse.builder().statusCode(SERVICE_UNAVAILABLE.getStatusCode()) + .payload(Payloads.newStringPayload(String.format("<Error><Code>%s</Code></Error>", code))).build(); + + expect(command.getCurrentRequest()).andReturn(putBucket); + final AtomicInteger counter = new AtomicInteger(); + expect(command.incrementFailureCount()).andAnswer(new IAnswer<Integer>() { + @Override + public Integer answer() throws Throwable { + return counter.incrementAndGet(); + } + }).anyTimes(); + expect(command.isReplayable()).andReturn(true).anyTimes(); + expect(command.getFailureCount()).andAnswer(new IAnswer<Integer>() { + @Override + public Integer answer() throws Throwable { + return counter.get(); + } + }).anyTimes(); + + AWSError error = new AWSError(); + error.setCode(code); + + expect(utils.parseAWSErrorFromContent(putBucket, limitExceeded)).andReturn(error); + + replay(utils, command); + + AWSServerErrorRetryHandler retry = new AWSServerErrorRetryHandler(utils, + ImmutableSet.<String> of("RequestLimitExceeded")); + + assert retry.shouldRetryRequest(command, limitExceeded); + + verify(utils, command); + + } + +}
