This is an automated email from the ASF dual-hosted git repository.
gaul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/jclouds.git
The following commit(s) were added to refs/heads/master by this push:
new 1c57d07 JCLOUDS-847: Poor upload performance for putBlob
1c57d07 is described below
commit 1c57d07f707be26cafc9d0da9abdc82af2827ad8
Author: Dori Polotsky <[email protected]>
AuthorDate: Mon Apr 22 22:55:09 2019 +0300
JCLOUDS-847: Poor upload performance for putBlob
This change improves the performance of writing to sockets with the
default Java URL connection HTTP client, by enlarging the buffer used
for socket writes from an implicit hard-coded 4KB / 8KB buffer to a
configurable 32KB buffer.
The buffer size is now controlled by the following property with the
following default value:
jclouds.output-socket-buffer-size: 32768
The implementation is based on a variant of ByteStreams.copy (written as
ByteStreams2.copy) which accepts the buffer size as an argument, unlike
the original Guava code that uses a hard-coded size.
The change was done directly within the loop that copies the input
stream to the output stream, and not by wrapping a BufferedOutputStream
around the existing output stream, in order to avoid copying the payload
twice.
On some platforms this change can improve both the putBlob throughput
and the total CPU consumption.
---
core/src/main/java/org/jclouds/Constants.java | 10 +++++++++-
.../org/jclouds/apis/internal/BaseApiMetadata.java | 2 ++
.../JavaUrlHttpCommandExecutorService.java | 8 ++++++--
.../src/main/java/org/jclouds/io/ByteStreams2.java | 23 ++++++++++++++++++++++
.../TrackingJavaUrlHttpCommandExecutorService.java | 4 +++-
.../dynect/v3/config/DynECTHttpApiModule.java | 4 +++-
...tatusFromPayloadHttpCommandExecutorService.java | 4 +++-
7 files changed, 49 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/org/jclouds/Constants.java
b/core/src/main/java/org/jclouds/Constants.java
index 388cc25..6c2143f 100644
--- a/core/src/main/java/org/jclouds/Constants.java
+++ b/core/src/main/java/org/jclouds/Constants.java
@@ -325,7 +325,15 @@ public final class Constants {
* </code>
*/
public static final String PROPERTY_TIMEOUTS_PREFIX = "jclouds.timeouts.";
-
+
+ /**
+ * Integer property. Default (32768).
+ * <p/>
+ * Buffer size for socket write (currently honored only by the default
+ * Java URL HTTP client, JavaUrlHttpCommandExcecutorService).
+ */
+ public static final String PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE =
"jclouds.output-socket-buffer-size";
+
/**
* Boolean property. Default (true).
* <p/>
diff --git a/core/src/main/java/org/jclouds/apis/internal/BaseApiMetadata.java
b/core/src/main/java/org/jclouds/apis/internal/BaseApiMetadata.java
index 5b85dc7..9202530 100644
--- a/core/src/main/java/org/jclouds/apis/internal/BaseApiMetadata.java
+++ b/core/src/main/java/org/jclouds/apis/internal/BaseApiMetadata.java
@@ -27,6 +27,7 @@ import static
org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST;
import static org.jclouds.Constants.PROPERTY_MAX_CONNECTION_REUSE;
import static org.jclouds.Constants.PROPERTY_MAX_PARALLEL_DELETES;
import static org.jclouds.Constants.PROPERTY_MAX_SESSION_FAILURES;
+import static org.jclouds.Constants.PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE;
import static org.jclouds.Constants.PROPERTY_PRETTY_PRINT_PAYLOADS;
import static org.jclouds.Constants.PROPERTY_SCHEDULER_THREADS;
import static org.jclouds.Constants.PROPERTY_SESSION_INTERVAL;
@@ -89,6 +90,7 @@ public abstract class BaseApiMetadata implements ApiMetadata {
props.setProperty(PROPERTY_MAX_PARALLEL_DELETES, numUserThreads + "");
props.setProperty(PROPERTY_IDEMPOTENT_METHODS,
"DELETE,GET,HEAD,OPTIONS,PUT");
+ props.setProperty(PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE, 32768 + "");
return props;
}
diff --git
a/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java
b/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java
index f64f4a7..1e455e2 100644
---
a/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java
+++
b/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java
@@ -22,6 +22,7 @@ import static
com.google.common.net.HttpHeaders.CONTENT_LENGTH;
import static com.google.common.net.HttpHeaders.HOST;
import static com.google.common.net.HttpHeaders.USER_AGENT;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
+import static org.jclouds.Constants.PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE;
import static org.jclouds.Constants.PROPERTY_USER_AGENT;
import static org.jclouds.http.HttpUtils.filterOutContentHeaders;
import static org.jclouds.io.Payloads.newInputStreamPayload;
@@ -50,6 +51,7 @@ import org.jclouds.http.HttpUtils;
import org.jclouds.http.IOExceptionRetryHandler;
import org.jclouds.http.handlers.DelegatingErrorHandler;
import org.jclouds.http.handlers.DelegatingRetryHandler;
+import org.jclouds.io.ByteStreams2;
import org.jclouds.io.ContentMetadataCodec;
import org.jclouds.io.MutableContentMetadata;
import org.jclouds.io.Payload;
@@ -58,7 +60,6 @@ import com.google.common.base.Function;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableMultimap.Builder;
-import com.google.common.io.ByteStreams;
import com.google.common.io.CountingOutputStream;
import com.google.inject.Inject;
@@ -69,6 +70,7 @@ public class JavaUrlHttpCommandExecutorService extends
BaseHttpCommandExecutorSe
protected final HostnameVerifier verifier;
@Inject(optional = true)
protected Supplier<SSLContext> sslContextSupplier;
+ protected final int outputSocketBufferSize;
protected final String userAgent;
@Inject
@@ -77,6 +79,7 @@ public class JavaUrlHttpCommandExecutorService extends
BaseHttpCommandExecutorSe
DelegatingErrorHandler errorHandler, HttpWire wire,
@Named("untrusted") HostnameVerifier verifier,
@Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider,
Function<URI, Proxy> proxyForURI,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods,
+ @Named(PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE) int outputSocketBufferSize,
@Named(PROPERTY_USER_AGENT) String userAgent) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler,
errorHandler, wire, idempotentMethods);
if (utils.getMaxConnections() > 0) {
@@ -86,6 +89,7 @@ public class JavaUrlHttpCommandExecutorService extends
BaseHttpCommandExecutorSe
this.verifier = checkNotNull(verifier, "verifier");
this.proxyForURI = checkNotNull(proxyForURI, "proxyForURI");
this.userAgent = userAgent;
+ this.outputSocketBufferSize = outputSocketBufferSize;
}
@Override
@@ -295,7 +299,7 @@ public class JavaUrlHttpCommandExecutorService extends
BaseHttpCommandExecutorSe
CountingOutputStream out = new
CountingOutputStream(connection.getOutputStream());
InputStream is = payload.openStream();
try {
- ByteStreams.copy(is, out);
+ ByteStreams2.copy(is, out, outputSocketBufferSize);
} catch (IOException e) {
logger.error(e, "error after writing %d/%s bytes to %s",
out.getCount(), lengthDesc, connection.getURL());
throw e;
diff --git a/core/src/main/java/org/jclouds/io/ByteStreams2.java
b/core/src/main/java/org/jclouds/io/ByteStreams2.java
index 814f1ec..44bd818 100644
--- a/core/src/main/java/org/jclouds/io/ByteStreams2.java
+++ b/core/src/main/java/org/jclouds/io/ByteStreams2.java
@@ -17,11 +17,13 @@
package org.jclouds.io;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.util.Closeables2.closeQuietly;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import com.google.common.annotations.Beta;
import com.google.common.hash.HashCode;
@@ -31,6 +33,8 @@ import com.google.common.io.ByteStreams;
@Beta
public class ByteStreams2 {
+ private static final int INPUT_STREAM_READ_END_OF_STREAM_INDICATOR = -1;
+
public static HashCode hashAndClose(InputStream input, HashFunction
hashFunction) throws IOException {
checkNotNull(input, "input");
checkNotNull(hashFunction, "hashFunction");
@@ -51,4 +55,23 @@ public class ByteStreams2 {
closeQuietly(input);
}
}
+
+ public static long copy(InputStream from, OutputStream to, int bufferSize)
throws IOException {
+ checkNotNull(from, "from");
+ checkNotNull(to, "to");
+ checkArgument(bufferSize >= 1, "bufferSize must be >= 1");
+
+ byte[] buf = new byte[bufferSize];
+ long total = 0L;
+
+ while (true) {
+ int len = from.read(buf);
+ if (len == INPUT_STREAM_READ_END_OF_STREAM_INDICATOR) {
+ return total;
+ }
+
+ to.write(buf, 0, len);
+ total += (long)len;
+ }
+ }
}
diff --git
a/core/src/test/java/org/jclouds/http/internal/TrackingJavaUrlHttpCommandExecutorService.java
b/core/src/test/java/org/jclouds/http/internal/TrackingJavaUrlHttpCommandExecutorService.java
index 35fe0ef..5b82765 100644
---
a/core/src/test/java/org/jclouds/http/internal/TrackingJavaUrlHttpCommandExecutorService.java
+++
b/core/src/test/java/org/jclouds/http/internal/TrackingJavaUrlHttpCommandExecutorService.java
@@ -17,6 +17,7 @@
package org.jclouds.http.internal;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
+import static org.jclouds.Constants.PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE;
import static org.jclouds.Constants.PROPERTY_USER_AGENT;
import java.net.Proxy;
@@ -92,10 +93,11 @@ public class TrackingJavaUrlHttpCommandExecutorService
extends JavaUrlHttpComman
@Named("untrusted") Supplier<SSLContext>
untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
List<HttpCommand> commandsInvoked,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods,
+ @Named(PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE) int
outputSocketBufferSize,
@Named(PROPERTY_USER_AGENT) String userAgent)
throws SecurityException, NoSuchFieldException {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler,
errorHandler, wire, verifier,
- untrustedSSLContextProvider, proxyForURI, idempotentMethods,
userAgent);
+ untrustedSSLContextProvider, proxyForURI, idempotentMethods,
outputSocketBufferSize, userAgent);
this.commandsInvoked = commandsInvoked;
}
diff --git
a/providers/dynect/src/main/java/org/jclouds/dynect/v3/config/DynECTHttpApiModule.java
b/providers/dynect/src/main/java/org/jclouds/dynect/v3/config/DynECTHttpApiModule.java
index a4ca194..5074b80 100644
---
a/providers/dynect/src/main/java/org/jclouds/dynect/v3/config/DynECTHttpApiModule.java
+++
b/providers/dynect/src/main/java/org/jclouds/dynect/v3/config/DynECTHttpApiModule.java
@@ -17,6 +17,7 @@
package org.jclouds.dynect.v3.config;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
+import static org.jclouds.Constants.PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE;
import static org.jclouds.Constants.PROPERTY_USER_AGENT;
import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
import static org.jclouds.rest.config.BinderUtils.bindHttpApi;
@@ -106,10 +107,11 @@ public class DynECTHttpApiModule extends
HttpApiModule<DynECTApi> {
DelegatingErrorHandler errorHandler, HttpWire wire,
@Named("untrusted") HostnameVerifier verifier,
@Named("untrusted") Supplier<SSLContext>
untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods,
+ @Named(PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE) int
outputSocketBufferSize,
@Named(PROPERTY_USER_AGENT) String userAgent)
throws SecurityException, NoSuchFieldException {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler,
errorHandler, wire, verifier,
- untrustedSSLContextProvider, proxyForURI, idempotentMethods,
userAgent);
+ untrustedSSLContextProvider, proxyForURI, idempotentMethods,
outputSocketBufferSize, userAgent);
}
/**
diff --git
a/providers/profitbricks/src/main/java/org/jclouds/profitbricks/http/ResponseStatusFromPayloadHttpCommandExecutorService.java
b/providers/profitbricks/src/main/java/org/jclouds/profitbricks/http/ResponseStatusFromPayloadHttpCommandExecutorService.java
index 4e337ff..afca237 100644
---
a/providers/profitbricks/src/main/java/org/jclouds/profitbricks/http/ResponseStatusFromPayloadHttpCommandExecutorService.java
+++
b/providers/profitbricks/src/main/java/org/jclouds/profitbricks/http/ResponseStatusFromPayloadHttpCommandExecutorService.java
@@ -17,6 +17,7 @@
package org.jclouds.profitbricks.http;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
+import static org.jclouds.Constants.PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE;
import static org.jclouds.Constants.PROPERTY_USER_AGENT;
import static org.jclouds.util.Closeables2.closeQuietly;
@@ -71,9 +72,10 @@ public class
ResponseStatusFromPayloadHttpCommandExecutorService extends JavaUrl
@Named("untrusted") Supplier<SSLContext>
untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
ParseSax<ServiceFault> faultHandler,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods,
+ @Named(PROPERTY_OUTPUT_SOCKET_BUFFER_SIZE) int
outputSocketBufferSize,
@Named(PROPERTY_USER_AGENT) String userAgent) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler,
errorHandler, wire, verifier, untrustedSSLContextProvider, proxyForURI,
- idempotentMethods, userAgent);
+ idempotentMethods, outputSocketBufferSize, userAgent);
this.faultHandler = faultHandler;
}
