This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a commit to branch no-channels
in repository https://gitbox.apache.org/repos/asf/maven-wagon.git
commit 466ca22e4faec9aca05727859232ace2075ff741
Author: Michael Osipov <1983-01...@gmx.net>
AuthorDate: Wed Jan 2 17:14:30 2019 +0100
Revert "[WAGON-537] Maven transfer speed of large artifacts is slow due to
unsuitable buffer strategy"
This reverts commit ff45ca83be7a2c07706a3a8dde420c897c58be1c.
---
.../java/org/apache/maven/wagon/AbstractWagon.java | 85 +++-----------------
.../wagon/shared/http/AbstractHttpClientWagon.java | 92 ++++++++--------------
2 files changed, 45 insertions(+), 132 deletions(-)
diff --git
a/wagon-provider-api/src/main/java/org/apache/maven/wagon/AbstractWagon.java
b/wagon-provider-api/src/main/java/org/apache/maven/wagon/AbstractWagon.java
index 8995559..4cbf37d 100644
--- a/wagon-provider-api/src/main/java/org/apache/maven/wagon/AbstractWagon.java
+++ b/wagon-provider-api/src/main/java/org/apache/maven/wagon/AbstractWagon.java
@@ -42,14 +42,8 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
import java.util.List;
-import static java.lang.Math.max;
-import static java.lang.Math.min;
-
/**
* Implementation of common facilities for Wagon providers.
*
@@ -58,24 +52,7 @@ import static java.lang.Math.min;
public abstract class AbstractWagon
implements Wagon
{
- protected static final int DEFAULT_BUFFER_SIZE = 4 * 1024;
- protected static final int MAXIMUM_BUFFER_SIZE = 512 * 1024;
-
- /**
- * To efficiently buffer data, use a multiple of 4 KiB as this is likely
to match the hardware
- * buffer size of certain storage devices.
- */
- protected static final int BUFFER_SEGMENT_SIZE = 4 * 1024;
-
- /**
- * The desired minimum amount of chunks in which a {@link Resource} shall
be
- * {@link #transfer(Resource, InputStream, OutputStream, int, long)
transferred}.
- * This corresponds to the minimum times {@link
#fireTransferProgress(TransferEvent, byte[], int)}
- * is executed. 100 notifications is a conservative value that will lead
to small chunks for
- * any artifact less that {@link #BUFFER_SEGMENT_SIZE} * {@link
#MINIMUM_AMOUNT_OF_TRANSFER_CHUNKS}
- * in size.
- */
- protected static final int MINIMUM_AMOUNT_OF_TRANSFER_CHUNKS = 100;
+ protected static final int DEFAULT_BUFFER_SIZE = 1024 * 4;
protected Repository repository;
@@ -583,71 +560,29 @@ public abstract class AbstractWagon
protected void transfer( Resource resource, InputStream input,
OutputStream output, int requestType, long maxSize )
throws IOException
{
-
- ByteBuffer buffer = ByteBuffer.allocate( getBufferCapacityForTransfer(
resource.getContentLength() ) );
- int halfBufferCapacity = buffer.capacity() / 2;
+ byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
TransferEvent transferEvent = new TransferEvent( this, resource,
TransferEvent.TRANSFER_PROGRESS, requestType );
transferEvent.setTimestamp( System.currentTimeMillis() );
- ReadableByteChannel in = Channels.newChannel( input );
-
long remaining = maxSize;
- while ( remaining > 0L )
+ while ( remaining > 0 )
{
- int read = in.read( buffer );
+ // let's safely cast to int because the min value will be lower
than the buffer size.
+ int n = input.read( buffer, 0, (int) Math.min( buffer.length,
remaining ) );
- if ( read == -1 )
+ if ( n == -1 )
{
- // EOF, but some data has not been written yet.
- if ( buffer.position() != 0 )
- {
- buffer.flip();
- fireTransferProgress( transferEvent, buffer.array(),
buffer.limit() );
- output.write( buffer.array(), 0, buffer.limit() );
- }
-
break;
}
- // Prevent minichunking / fragmentation: when less than half the
buffer is utilized,
- // read some more bytes before writing and firing progress.
- if ( buffer.position() < halfBufferCapacity )
- {
- continue;
- }
+ fireTransferProgress( transferEvent, buffer, n );
- buffer.flip();
- fireTransferProgress( transferEvent, buffer.array(),
buffer.limit() );
- output.write( buffer.array(), 0, buffer.limit() );
- remaining -= buffer.limit();
- buffer.clear();
- }
- output.flush();
- }
+ output.write( buffer, 0, n );
- /**
- * Provides a buffer size for efficiently transferring the given amount of
bytes such that
- * it is not fragmented into too many chunks. For larger files larger
buffers are provided such that downstream
- * {@link #fireTransferProgress(TransferEvent, byte[], int) listeners} are
not notified too frequently.
- * For instance, transferring gigabyte-sized resources would result in
millions of notifications when using
- * only a few kibibytes of buffer, drastically slowing down transfer since
transfer progress listeners and
- * notifications are synchronous and may block, e.g., when writing
download progress status to console.
- *
- * @param numberOfBytes can be 0 or less, in which case a default buffer
size is used.
- * @return a byte buffer suitable for transferring the given amount of
bytes without too many chunks.
- */
- protected int getBufferCapacityForTransfer( long numberOfBytes )
- {
- if ( numberOfBytes <= 0L )
- {
- return DEFAULT_BUFFER_SIZE;
+ remaining -= n;
}
-
- final int numberOfBufferSegments = (int)
- numberOfBytes / ( BUFFER_SEGMENT_SIZE *
MINIMUM_AMOUNT_OF_TRANSFER_CHUNKS );
- final int potentialBufferSize = numberOfBufferSegments *
BUFFER_SEGMENT_SIZE;
- return min( MAXIMUM_BUFFER_SIZE, max( DEFAULT_BUFFER_SIZE,
potentialBufferSize ) );
+ output.flush();
}
// ----------------------------------------------------------------------
diff --git
a/wagon-providers/wagon-http-shared/src/main/java/org/apache/maven/wagon/shared/http/AbstractHttpClientWagon.java
b/wagon-providers/wagon-http-shared/src/main/java/org/apache/maven/wagon/shared/http/AbstractHttpClientWagon.java
index 5b909bb..9f294f7 100755
---
a/wagon-providers/wagon-http-shared/src/main/java/org/apache/maven/wagon/shared/http/AbstractHttpClientWagon.java
+++
b/wagon-providers/wagon-http-shared/src/main/java/org/apache/maven/wagon/shared/http/AbstractHttpClientWagon.java
@@ -25,7 +25,6 @@ import org.apache.http.HttpException;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
-import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.ChallengeState;
import org.apache.http.auth.Credentials;
@@ -34,7 +33,6 @@ import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
@@ -54,9 +52,6 @@ import org.apache.http.conn.ssl.SSLContextBuilder;
import org.apache.http.conn.ssl.SSLInitializationException;
import org.apache.http.entity.AbstractHttpEntity;
import org.apache.http.impl.auth.BasicScheme;
-import org.apache.http.impl.auth.BasicSchemeFactory;
-import org.apache.http.impl.auth.DigestSchemeFactory;
-import org.apache.http.impl.auth.NTLMSchemeFactory;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -89,11 +84,6 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
@@ -115,6 +105,9 @@ public abstract class AbstractHttpClientWagon
private final class RequestEntityImplementation
extends AbstractHttpEntity
{
+
+ private static final int BUFFER_SIZE = 2048;
+
private final Resource resource;
private final Wagon wagon;
@@ -167,57 +160,52 @@ public abstract class AbstractHttpClientWagon
return repeatable;
}
- public void writeTo( final OutputStream output )
+ public void writeTo( final OutputStream outputStream )
throws IOException
{
- if ( output == null )
+ if ( outputStream == null )
{
- throw new NullPointerException( "output cannot be null" );
+ throw new NullPointerException( "outputStream cannot be null"
);
}
TransferEvent transferEvent =
new TransferEvent( wagon, resource,
TransferEvent.TRANSFER_PROGRESS, TransferEvent.REQUEST_PUT );
transferEvent.setTimestamp( System.currentTimeMillis() );
-
- try ( ReadableByteChannel input = ( this.source != null )
- ? new RandomAccessFile( this.source, "r" ).getChannel()
- : Channels.newChannel( stream ) )
+ InputStream instream = ( this.source != null )
+ ? new FileInputStream( this.source )
+ : stream;
+ try
{
- ByteBuffer buffer = ByteBuffer.allocate(
getBufferCapacityForTransfer( this.length ) );
- int halfBufferCapacity = buffer.capacity() / 2;
-
- long remaining = this.length < 0L ? Long.MAX_VALUE :
this.length;
- while ( remaining > 0L )
+ byte[] buffer = new byte[BUFFER_SIZE];
+ int l;
+ if ( this.length < 0 )
{
- int read = input.read( buffer );
- if ( read == -1 )
+ // until EOF
+ while ( ( l = instream.read( buffer ) ) != -1 )
{
- // EOF, but some data has not been written yet.
- if ( buffer.position() != 0 )
- {
- buffer.flip();
- fireTransferProgress( transferEvent,
buffer.array(), buffer.limit() );
- output.write( buffer.array(), 0, buffer.limit() );
- buffer.clear();
- }
-
- break;
+ fireTransferProgress( transferEvent, buffer, -1 );
+ outputStream.write( buffer, 0, l );
}
-
- // Prevent minichunking / fragmentation: when less than
half the buffer is utilized,
- // read some more bytes before writing and firing progress.
- if ( buffer.position() < halfBufferCapacity )
+ }
+ else
+ {
+ // no need to consume more than length
+ long remaining = this.length;
+ while ( remaining > 0 )
{
- continue;
+ l = instream.read( buffer, 0, (int) Math.min(
BUFFER_SIZE, remaining ) );
+ if ( l == -1 )
+ {
+ break;
+ }
+ fireTransferProgress( transferEvent, buffer, (int)
Math.min( BUFFER_SIZE, remaining ) );
+ outputStream.write( buffer, 0, l );
+ remaining -= l;
}
-
- buffer.flip();
- fireTransferProgress( transferEvent, buffer.array(),
buffer.limit() );
- output.write( buffer.array(), 0, buffer.limit() );
- remaining -= buffer.limit();
- buffer.clear();
-
}
- output.flush();
+ }
+ finally
+ {
+ instream.close();
}
}
@@ -451,15 +439,6 @@ public abstract class AbstractHttpClientWagon
}
}
- private static Registry<AuthSchemeProvider> createAuthSchemeRegistry()
- {
- return RegistryBuilder.<AuthSchemeProvider>create()
- .register( AuthSchemes.BASIC, new BasicSchemeFactory(
StandardCharsets.UTF_8 ) )
- .register( AuthSchemes.DIGEST, new DigestSchemeFactory(
StandardCharsets.UTF_8 ) )
- .register( AuthSchemes.NTLM, new NTLMSchemeFactory() )
- .build();
- }
-
private static Collection<Class<? extends IOException>>
getNonRetryableExceptions()
{
final List<Class<? extends IOException>> exceptions = new
ArrayList<>();
@@ -487,7 +466,6 @@ public abstract class AbstractHttpClientWagon
.disableConnectionState() //
.setConnectionManager( httpClientConnectionManager ) //
.setRetryHandler( createRetryHandler() )
- .setDefaultAuthSchemeRegistry( createAuthSchemeRegistry() )
.build();
}
