Repository: maven-wagon Updated Branches: refs/heads/master 644b814d1 -> 4c5df03f4
Add HTTP429 exponential backoff to the httpclient version of wagon Project: http://git-wip-us.apache.org/repos/asf/maven-wagon/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-wagon/commit/8add903e Tree: http://git-wip-us.apache.org/repos/asf/maven-wagon/tree/8add903e Diff: http://git-wip-us.apache.org/repos/asf/maven-wagon/diff/8add903e Branch: refs/heads/master Commit: 8add903e12106cbeea7c15039013e9b46cc98b2b Parents: 5235dac Author: Michael Neale <[email protected]> Authored: Wed Sep 3 16:14:45 2014 +1000 Committer: Michael Neale <[email protected]> Committed: Thu Sep 4 18:13:05 2014 +1000 ---------------------------------------------------------------------- .../maven/wagon/http/HttpWagonTestCase.java | 177 ++++++++++++++++++- .../providers/http/AbstractHttpClientWagon.java | 94 +++++++++- .../maven/wagon/providers/http/HttpWagon.java | 18 +- 3 files changed, 284 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-wagon/blob/8add903e/wagon-provider-test/src/main/java/org/apache/maven/wagon/http/HttpWagonTestCase.java ---------------------------------------------------------------------- diff --git a/wagon-provider-test/src/main/java/org/apache/maven/wagon/http/HttpWagonTestCase.java b/wagon-provider-test/src/main/java/org/apache/maven/wagon/http/HttpWagonTestCase.java index b7432e3..e20dc40 100644 --- a/wagon-provider-test/src/main/java/org/apache/maven/wagon/http/HttpWagonTestCase.java +++ b/wagon-provider-test/src/main/java/org/apache/maven/wagon/http/HttpWagonTestCase.java @@ -68,6 +68,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.zip.GZIPOutputStream; /** @@ -244,6 +245,67 @@ public abstract class HttpWagonTestCase } } + public void testList429() + throws Exception + { + StreamingWagon wagon = (StreamingWagon) getWagon(); + try + { + + Server server = new Server( 0 ); + final AtomicBoolean called = new AtomicBoolean(); + + AbstractHandler handler = new AbstractHandler() { + public void handle(String s, HttpServletRequest request, HttpServletResponse response, int i) throws IOException, ServletException + { + if (called.get()) + { + response.setStatus( 200 ); + ( (Request) request ).setHandled( true ); + } + else + { + called.set(true); + response.setStatus( 429 ); + ( (Request) request ).setHandled( true ); + + } + } + }; + + server.setHandler( handler ); + addConnectors( server ); + server.start(); + + wagon.connect( new Repository( "id", getRepositoryUrl( server ) ) ); + + try + { + wagon.getFileList("resource"); + } + finally + { + wagon.disconnect(); + + server.stop(); + } + + } + catch ( ResourceDoesNotExistException e ) + { + assertTrue( true ); + } + catch ( TransferFailedException e ) { + if (wagon.getClass().getName().contains("Lightweight")) { + //we don't care about lightweight + assertTrue( true ); + } else { + fail(); + } + + } + } + public void testGet500() throws Exception { @@ -254,7 +316,7 @@ public abstract class HttpWagonTestCase } catch ( TransferFailedException e ) { - assertTrue( true ); + assertTrue(true); } } @@ -274,7 +336,7 @@ public abstract class HttpWagonTestCase try { - wagon.getToStream( "resource", new ByteArrayOutputStream() ); + wagon.getToStream("resource", new ByteArrayOutputStream()); fail(); } finally @@ -326,6 +388,59 @@ public abstract class HttpWagonTestCase } } + public void testResourceExists429() + throws Exception + { + try + { + + final AtomicBoolean called = new AtomicBoolean(); + + AbstractHandler handler = new AbstractHandler() { + public void handle(String s, HttpServletRequest request, HttpServletResponse response, int i) + throws IOException, ServletException + { + if (called.get()) + { + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + ((Request) request).setHandled(true); + } else + { + called.set(true); + response.setStatus(429); + ((Request) request).setHandled(true); + } + } + }; + + StreamingWagon wagon = (StreamingWagon) getWagon(); + Server server = new Server( 0 ); + server.setHandler( handler ); + addConnectors( server ); + server.start(); + wagon.connect( new Repository( "id", getRepositoryUrl( server ) ) ); + + try + { + wagon.resourceExists( "resource" ); + } + finally + { + wagon.disconnect(); + + server.stop(); + } + + + fail(); + } + catch ( TransferFailedException e ) + { + assertTrue( true ); + } + } + + private boolean runTestResourceExists( int status ) throws Exception { @@ -1294,6 +1409,64 @@ public abstract class HttpWagonTestCase } } + public void testPut429() + throws Exception + { + + try { + + + StreamingWagon wagon = (StreamingWagon) getWagon(); + Server server = new Server(0); + final AtomicBoolean called = new AtomicBoolean(); + + AbstractHandler handler = new AbstractHandler() { + public void handle(String s, HttpServletRequest request, HttpServletResponse response, int i) + throws IOException, ServletException + { + if (called.get()) + { + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + ((Request) request).setHandled(true); + } else + { + called.set(true); + response.setStatus(429); + ((Request) request).setHandled(true); + } + } + }; + + server.setHandler(handler); + addConnectors(server); + server.start(); + + wagon.connect(new Repository("id", getRepositoryUrl(server))); + + File tempFile = File.createTempFile("wagon", "tmp"); + tempFile.deleteOnExit(); + FileUtils.fileWrite(tempFile.getAbsolutePath(), "content"); + + try + { + wagon.put(tempFile, "resource"); + fail(); + } finally + { + wagon.disconnect(); + + server.stop(); + + tempFile.delete(); + } + + } catch ( TransferFailedException e ) + { + assertTrue( true ); + } + } + + private void runTestPut( int status ) throws Exception { http://git-wip-us.apache.org/repos/asf/maven-wagon/blob/8add903e/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/AbstractHttpClientWagon.java ---------------------------------------------------------------------- diff --git a/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/AbstractHttpClientWagon.java b/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/AbstractHttpClientWagon.java index 7c6dcd2..8efa6b3 100755 --- a/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/AbstractHttpClientWagon.java +++ b/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/AbstractHttpClientWagon.java @@ -268,6 +268,47 @@ public abstract class AbstractHttpClientWagon */ private static final PoolingHttpClientConnectionManager CONN_MAN = createConnManager(); + + + /** + * See RFC6585 + */ + protected static final int SC_TOO_MANY_REQUESTS = 429; + + /** + * For exponential backoff. + */ + + /** + * Initial seconds to back off when a HTTP 429 received. + * Subsequent 429 responses result in exponental backoff. + * <b>5 by default</b> + */ + protected final static int INITIAL_BACKOFF_SECONDS = + Integer.parseInt( System.getProperty( "maven.wagon.httpconnectionManager.backoffSeconds", "5" ) ); + + /** + * The maximum amount of time we want to back off in the case of + * repeated HTTP 429 response codes. + */ + protected final static int MAX_WAIT_SECONDS = + Integer.parseInt( System.getProperty( "maven.wagon.httpconnectionManager.maxBackoffSeconds", "180" ) ); + + + + protected int backoff(int wait, String url) throws InterruptedException, TransferFailedException + { + TimeUnit.SECONDS.sleep(wait); + int nextWait = wait * 2; + if (nextWait >= MAX_WAIT_SECONDS) + { + throw new TransferFailedException( + "Waited too long to access: " + url + ". Return code is: " + SC_TOO_MANY_REQUESTS); + } + return nextWait; + } + + private static PoolingHttpClientConnectionManager createConnManager() { @@ -493,8 +534,16 @@ public abstract class AbstractHttpClientWagon put( resource, source, httpEntity, url.toString() ); } - private void put( Resource resource, File source, HttpEntity httpEntity, String url ) - throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException + + private void put(Resource resource, File source, HttpEntity httpEntity, String url ) + throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException + { + put(INITIAL_BACKOFF_SECONDS, resource, source, httpEntity, url); + } + + + private void put(int wait, Resource resource, File source, HttpEntity httpEntity, String url ) + throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException { //Parent directories need to be created before posting @@ -567,6 +616,9 @@ public abstract class AbstractHttpClientWagon case HttpStatus.SC_NOT_FOUND: throw new ResourceDoesNotExistException( "File: " + url + " does not exist" + reasonPhrase ); + case SC_TOO_MANY_REQUESTS: + put(backoff(wait, url), resource, source, httpEntity, url); + break; //add more entries here default: { @@ -598,6 +650,13 @@ public abstract class AbstractHttpClientWagon throw new TransferFailedException( e.getMessage(), e ); } + catch ( InterruptedException e ) + { + fireTransferError( resource, e, TransferEvent.REQUEST_PUT ); + + throw new TransferFailedException( e.getMessage(), e ); + } + } protected String calculateRelocatedUrl( HttpResponse response ) @@ -615,6 +674,12 @@ public abstract class AbstractHttpClientWagon } public boolean resourceExists( String resourceName ) + throws TransferFailedException, AuthorizationException { + return resourceExists( INITIAL_BACKOFF_SECONDS, resourceName); + } + + + private boolean resourceExists(int wait, String resourceName ) throws TransferFailedException, AuthorizationException { String repositoryUrl = getRepository().getUrl(); @@ -648,6 +713,10 @@ public abstract class AbstractHttpClientWagon case HttpStatus.SC_NOT_FOUND: result = false; break; + + case SC_TOO_MANY_REQUESTS: + return resourceExists(backoff(wait, resourceName), resourceName); + //add more entries here default: throw new TransferFailedException( @@ -670,6 +739,11 @@ public abstract class AbstractHttpClientWagon { throw new TransferFailedException( e.getMessage(), e ); } + catch ( InterruptedException e ) + { + throw new TransferFailedException( e.getMessage(), e ); + } + } protected CloseableHttpResponse execute( HttpUriRequest httpMethod ) @@ -832,6 +906,11 @@ public abstract class AbstractHttpClientWagon } public void fillInputData( InputData inputData ) + throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException { + fillInputData(INITIAL_BACKOFF_SECONDS, inputData); + } + + private void fillInputData(int wait, InputData inputData ) throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException { Resource resource = inputData.getResource(); @@ -882,6 +961,10 @@ public abstract class AbstractHttpClientWagon case HttpStatus.SC_NOT_FOUND: throw new ResourceDoesNotExistException( "File: " + url + " " + reasonPhrase ); + case SC_TOO_MANY_REQUESTS: + fillInputData(backoff(wait, url), inputData); + break; + // add more entries here default: { @@ -940,6 +1023,13 @@ public abstract class AbstractHttpClientWagon throw new TransferFailedException( e.getMessage(), e ); } + catch ( InterruptedException e ) + { + fireTransferError( resource, e, TransferEvent.REQUEST_GET ); + + throw new TransferFailedException( e.getMessage(), e ); + } + } protected void cleanupGetTransfer( Resource resource ) http://git-wip-us.apache.org/repos/asf/maven-wagon/blob/8add903e/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/HttpWagon.java ---------------------------------------------------------------------- diff --git a/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/HttpWagon.java b/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/HttpWagon.java index e307e40..2048937 100755 --- a/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/HttpWagon.java +++ b/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/HttpWagon.java @@ -22,6 +22,7 @@ package org.apache.maven.wagon.providers.http; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.http.HttpEntity; import org.apache.http.HttpException; @@ -40,7 +41,14 @@ import org.apache.maven.wagon.shared.http.HtmlFileListParser; public class HttpWagon extends AbstractHttpClientWagon { - public List<String> getFileList( String destinationDirectory ) + + public List<String> getFileList(String destinationDirectory ) + throws AuthorizationException, ResourceDoesNotExistException, TransferFailedException + { + return getFileList(INITIAL_BACKOFF_SECONDS, destinationDirectory); + } + + private List<String> getFileList(int wait, String destinationDirectory ) throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException { if ( destinationDirectory.length() > 0 && !destinationDirectory.endsWith( "/" ) ) @@ -77,6 +85,9 @@ public class HttpWagon case HttpStatus.SC_NOT_FOUND: throw new ResourceDoesNotExistException( "File: " + url + " does not exist" ); + case SC_TOO_MANY_REQUESTS: + return getFileList(backoff(wait, url), destinationDirectory); + //add more entries here default: throw new TransferFailedException( @@ -106,5 +117,10 @@ public class HttpWagon { throw new TransferFailedException( "Could not read response body.", e ); } + catch ( InterruptedException e ) + { + throw new TransferFailedException( "Unable to wait for resource.", e ); + } } + }
