Author: pmouawad
Date: Wed Mar 9 12:23:31 2016
New Revision: 1734228
URL: http://svn.apache.org/viewvc?rev=1734228&view=rev
Log:
[Bug 52073] Embedded Resources Parallel download : Improve performances by
avoiding shutdown of ThreadPoolExecutor at each sample
Based on PR by Benoit Wiart + the addition (blame me) of
JMeterPoolingClientConnectionManager (see mailing list mail I will send)
Bugzilla Id: 52073
Added:
jmeter/trunk/src/protocol/http/org/apache/http/
jmeter/trunk/src/protocol/http/org/apache/http/impl/
jmeter/trunk/src/protocol/http/org/apache/http/impl/conn/
jmeter/trunk/src/protocol/http/org/apache/http/impl/conn/JMeterPoolingClientConnectionManager.java
(with props)
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/ResourcesDownloader.java
(with props)
Modified:
jmeter/trunk/bin/jmeter.properties
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/control/CacheManager.java
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/MeasuringConnectionManager.java
jmeter/trunk/xdocs/changes.xml
Modified: jmeter/trunk/bin/jmeter.properties
URL:
http://svn.apache.org/viewvc/jmeter/trunk/bin/jmeter.properties?rev=1734228&r1=1734227&r2=1734228&view=diff
==============================================================================
--- jmeter/trunk/bin/jmeter.properties (original)
+++ jmeter/trunk/bin/jmeter.properties Wed Mar 9 12:23:31 2016
@@ -468,11 +468,18 @@ log_level.jorphan=INFO
# Number of retries to attempt (default 0)
#httpclient4.retrycount=0
-# Idle connection timeout (ms) to apply if the server does not send
+# Idle connection timeout (Milliseconds) to apply if the server does not send
# Keep-Alive headers (default 0 = no Keep-Alive)
#httpclient4.idletimeout=0
# Note: this is currently an experimental fix
+# Check connections if the elapsed time (Milliseconds) since the last
+# use of the connection exceed this value
+#httpclient4.validate_after_inactivity=2000
+
+# TTL (in Milliseconds) represents an absolute value.
+# No matter what the connection will not be re-used beyond its TTL.
+#httpclient4.time_to_live=2000
#---------------------------------------------------------------------------
# Apache HttpComponents HTTPClient configuration (HTTPClient 3.1)
#---------------------------------------------------------------------------
@@ -992,8 +999,7 @@ beanshell.server.file=../extras/startup.
#httpsampler.max_redirects=5
# Maximum frame/iframe nesting depth (default 5)
#httpsampler.max_frame_depth=5
-# Maximum await termination timeout (secs) when concurrent download embedded
resources (default 60)
-#httpsampler.await_termination_timeout=60
+
# Revert to BUG 51939 behaviour (no separate container for embedded resources)
by setting the following false:
#httpsampler.separate.container=true
@@ -1001,6 +1007,9 @@ beanshell.server.file=../extras/startup.
# Parent sample will not be marked as failed
#httpsampler.ignore_failed_embedded_resources=false
+#keep alive time for the parallel download threads (in seconds)
+#httpsampler.parallel_download_thread_keepalive_inseconds=60
+
# Don't keep the embedded resources response data : just keep the size and the
md5
# default to false
#httpsampler.embedded_resources_use_md5=false
Added:
jmeter/trunk/src/protocol/http/org/apache/http/impl/conn/JMeterPoolingClientConnectionManager.java
URL:
http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/http/impl/conn/JMeterPoolingClientConnectionManager.java?rev=1734228&view=auto
==============================================================================
---
jmeter/trunk/src/protocol/http/org/apache/http/impl/conn/JMeterPoolingClientConnectionManager.java
(added)
+++
jmeter/trunk/src/protocol/http/org/apache/http/impl/conn/JMeterPoolingClientConnectionManager.java
Wed Mar 9 12:23:31 2016
@@ -0,0 +1,353 @@
+/*
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation. For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+package org.apache.http.impl.conn;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.annotation.ThreadSafe;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.conn.ClientConnectionOperator;
+import org.apache.http.conn.ClientConnectionRequest;
+import org.apache.http.conn.ConnectionPoolTimeoutException;
+import org.apache.http.conn.DnsResolver;
+import org.apache.http.conn.ManagedClientConnection;
+import org.apache.http.conn.routing.HttpRoute;
+import org.apache.http.conn.scheme.SchemeRegistry;
+import org.apache.http.pool.ConnPoolControl;
+import org.apache.http.pool.PoolStats;
+import org.apache.http.util.Args;
+import org.apache.http.util.Asserts;
+
+/**
+ * Copy/Paste of {@link PoolingClientConnectionManager} with a slight
modification
+ * extracted from {@link PoolingHttpClientConnectionManager} to allow using
+ * better validation mechanism introduced in 4.4
+ * TODO : Remove when full upgrade to new HttpClient 4.5.X API is finished
+ * @deprecated Will be removed in 3.1, DO NOT USE
+ */
+@Deprecated
+@ThreadSafe
+public class JMeterPoolingClientConnectionManager implements
ClientConnectionManager, ConnPoolControl<HttpRoute> {
+
+ private static final int VALIDATE_AFTER_INACTIVITY_DEFAULT = 2000;
+
+ private final Log log = LogFactory.getLog(getClass());
+
+ private final SchemeRegistry schemeRegistry;
+
+ private final HttpConnPool pool;
+
+ private final ClientConnectionOperator operator;
+
+
+ /** the custom-configured DNS lookup mechanism. */
+ private final DnsResolver dnsResolver;
+
+ public JMeterPoolingClientConnectionManager(final SchemeRegistry schreg) {
+ this(schreg, -1, TimeUnit.MILLISECONDS);
+ }
+
+ public JMeterPoolingClientConnectionManager(final SchemeRegistry
schreg,final DnsResolver dnsResolver) {
+ this(schreg, -1, TimeUnit.MILLISECONDS,dnsResolver);
+ }
+
+ public JMeterPoolingClientConnectionManager() {
+ this(SchemeRegistryFactory.createDefault());
+ }
+
+ public JMeterPoolingClientConnectionManager(
+ final SchemeRegistry schemeRegistry,
+ final long timeToLive, final TimeUnit tunit) {
+ this(schemeRegistry, timeToLive, tunit, new
SystemDefaultDnsResolver());
+ }
+
+ public JMeterPoolingClientConnectionManager(
+ final SchemeRegistry schemeRegistry,
+ final long timeToLive, final TimeUnit tunit,
+ final DnsResolver dnsResolver) {
+ this(schemeRegistry, timeToLive, tunit, dnsResolver,
VALIDATE_AFTER_INACTIVITY_DEFAULT);
+ }
+
+ public JMeterPoolingClientConnectionManager(final SchemeRegistry
schemeRegistry,
+ final long timeToLive, final TimeUnit tunit,
+ final DnsResolver dnsResolver,
+ int validateAfterInactivity) {
+ super();
+ Args.notNull(schemeRegistry, "Scheme registry");
+ Args.notNull(dnsResolver, "DNS resolver");
+ this.schemeRegistry = schemeRegistry;
+ this.dnsResolver = dnsResolver;
+ this.operator = createConnectionOperator(schemeRegistry);
+ this.pool = new HttpConnPool(this.log, this.operator, 2, 20,
timeToLive, tunit);
+ pool.setValidateAfterInactivity(validateAfterInactivity);
+ }
+
+ /**
+ * @see #setValidateAfterInactivity(int)
+ *
+ * @since 4.5.2
+ */
+ public int getValidateAfterInactivity() {
+ return pool.getValidateAfterInactivity();
+ }
+
+ /**
+ * Defines period of inactivity in milliseconds after which persistent
connections must
+ * be re-validated prior to being {@link
#leaseConnection(java.util.concurrent.Future,
+ * long, java.util.concurrent.TimeUnit) leased} to the consumer.
Non-positive value passed
+ * to this method disables connection validation. This check helps detect
connections
+ * that have become stale (half-closed) while kept inactive in the pool.
+ *
+ * @see #leaseConnection(java.util.concurrent.Future, long,
java.util.concurrent.TimeUnit)
+ *
+ * @since 4.5.2
+ */
+ public void setValidateAfterInactivity(final int ms) {
+ pool.setValidateAfterInactivity(ms);
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ shutdown();
+ } finally {
+ super.finalize();
+ }
+ }
+
+ /**
+ * Hook for creating the connection operator.
+ * It is called by the constructor.
+ * Derived classes can override this method to change the
+ * instantiation of the operator.
+ * The default implementation here instantiates
+ * {@link DefaultClientConnectionOperator DefaultClientConnectionOperator}.
+ *
+ * @param schreg the scheme registry.
+ *
+ * @return the connection operator to use
+ */
+ protected ClientConnectionOperator createConnectionOperator(final
SchemeRegistry schreg) {
+ return new DefaultClientConnectionOperator(schreg,
this.dnsResolver);
+ }
+ @Override
+ public SchemeRegistry getSchemeRegistry() {
+ return this.schemeRegistry;
+ }
+
+ private String format(final HttpRoute route, final Object state) {
+ final StringBuilder buf = new StringBuilder();
+ buf.append("[route: ").append(route).append("]");
+ if (state != null) {
+ buf.append("[state: ").append(state).append("]");
+ }
+ return buf.toString();
+ }
+
+ private String formatStats(final HttpRoute route) {
+ final StringBuilder buf = new StringBuilder();
+ final PoolStats totals = this.pool.getTotalStats();
+ final PoolStats stats = this.pool.getStats(route);
+ buf.append("[total kept alive:
").append(totals.getAvailable()).append("; ");
+ buf.append("route allocated: ").append(stats.getLeased() +
stats.getAvailable());
+ buf.append(" of ").append(stats.getMax()).append("; ");
+ buf.append("total allocated: ").append(totals.getLeased() +
totals.getAvailable());
+ buf.append(" of ").append(totals.getMax()).append("]");
+ return buf.toString();
+ }
+
+ private String format(final HttpPoolEntry entry) {
+ final StringBuilder buf = new StringBuilder();
+ buf.append("[id: ").append(entry.getId()).append("]");
+ buf.append("[route: ").append(entry.getRoute()).append("]");
+ final Object state = entry.getState();
+ if (state != null) {
+ buf.append("[state: ").append(state).append("]");
+ }
+ return buf.toString();
+ }
+
+ @Override
+ public ClientConnectionRequest requestConnection(
+ final HttpRoute route,
+ final Object state) {
+ Args.notNull(route, "HTTP route");
+ if (this.log.isDebugEnabled()) {
+ this.log.debug("Connection request: " + format(route, state) +
formatStats(route));
+ }
+ final Future<HttpPoolEntry> future = this.pool.lease(route, state);
+
+ return new ClientConnectionRequest() {
+ @Override
+ public void abortRequest() {
+ future.cancel(true);
+ }
+ @Override
+ public ManagedClientConnection getConnection(
+ final long timeout,
+ final TimeUnit tunit) throws InterruptedException,
ConnectionPoolTimeoutException {
+ return leaseConnection(future, timeout, tunit);
+ }
+
+ };
+
+ }
+
+ ManagedClientConnection leaseConnection(
+ final Future<HttpPoolEntry> future,
+ final long timeout,
+ final TimeUnit tunit) throws InterruptedException,
ConnectionPoolTimeoutException {
+ final HttpPoolEntry entry;
+ try {
+ entry = future.get(timeout, tunit);
+ if (entry == null || future.isCancelled()) {
+ throw new InterruptedException();
+ }
+ Asserts.check(entry.getConnection() != null, "Pool entry with no
connection");
+ if (this.log.isDebugEnabled()) {
+ this.log.debug("Connection leased: " + format(entry) +
formatStats(entry.getRoute()));
+ }
+ return new ManagedClientConnectionImpl(this, this.operator, entry);
+ } catch (final ExecutionException ex) {
+ Throwable cause = ex.getCause();
+ if (cause == null) {
+ cause = ex;
+ }
+ this.log.error("Unexpected exception leasing connection from
pool", cause);
+ // Should never happen
+ throw new InterruptedException();
+ } catch (final TimeoutException ex) {
+ throw new ConnectionPoolTimeoutException("Timeout waiting for
connection from pool");
+ }
+ }
+ @Override
+ public void releaseConnection(
+ final ManagedClientConnection conn, final long keepalive, final
TimeUnit tunit) {
+
+ Args.check(conn instanceof ManagedClientConnectionImpl, "Connection
class mismatch, " +
+ "connection not obtained from this manager");
+ final ManagedClientConnectionImpl managedConn =
(ManagedClientConnectionImpl) conn;
+ Asserts.check(managedConn.getManager() == this, "Connection not
obtained from this manager");
+ synchronized (managedConn) {
+ final HttpPoolEntry entry = managedConn.detach();
+ if (entry == null) {
+ return;
+ }
+ try {
+ if (managedConn.isOpen() && !managedConn.isMarkedReusable()) {
+ try {
+ managedConn.shutdown();
+ } catch (final IOException iox) {
+ if (this.log.isDebugEnabled()) {
+ this.log.debug("I/O exception shutting down
released connection", iox);
+ }
+ }
+ }
+ // Only reusable connections can be kept alive
+ if (managedConn.isMarkedReusable()) {
+ entry.updateExpiry(keepalive, tunit != null ? tunit :
TimeUnit.MILLISECONDS);
+ if (this.log.isDebugEnabled()) {
+ final String s;
+ if (keepalive > 0) {
+ s = "for " + keepalive + " " + tunit;
+ } else {
+ s = "indefinitely";
+ }
+ this.log.debug("Connection " + format(entry) + " can
be kept alive " + s);
+ }
+ }
+ } finally {
+ this.pool.release(entry, managedConn.isMarkedReusable());
+ }
+ if (this.log.isDebugEnabled()) {
+ this.log.debug("Connection released: " + format(entry) +
formatStats(entry.getRoute()));
+ }
+ }
+ }
+ @Override
+ public void shutdown() {
+ this.log.debug("Connection manager is shutting down");
+ try {
+ this.pool.shutdown();
+ } catch (final IOException ex) {
+ this.log.debug("I/O exception shutting down connection manager",
ex);
+ }
+ this.log.debug("Connection manager shut down");
+ }
+ @Override
+ public void closeIdleConnections(final long idleTimeout, final TimeUnit
tunit) {
+ if (this.log.isDebugEnabled()) {
+ this.log.debug("Closing connections idle longer than " +
idleTimeout + " " + tunit);
+ }
+ this.pool.closeIdle(idleTimeout, tunit);
+ }
+ @Override
+ public void closeExpiredConnections() {
+ this.log.debug("Closing expired connections");
+ this.pool.closeExpired();
+ }
+ @Override
+ public int getMaxTotal() {
+ return this.pool.getMaxTotal();
+ }
+ @Override
+ public void setMaxTotal(final int max) {
+ this.pool.setMaxTotal(max);
+ }
+ @Override
+ public int getDefaultMaxPerRoute() {
+ return this.pool.getDefaultMaxPerRoute();
+ }
+ @Override
+ public void setDefaultMaxPerRoute(final int max) {
+ this.pool.setDefaultMaxPerRoute(max);
+ }
+ @Override
+ public int getMaxPerRoute(final HttpRoute route) {
+ return this.pool.getMaxPerRoute(route);
+ }
+ @Override
+ public void setMaxPerRoute(final HttpRoute route, final int max) {
+ this.pool.setMaxPerRoute(route, max);
+ }
+ @Override
+ public PoolStats getTotalStats() {
+ return this.pool.getTotalStats();
+ }
+ @Override
+ public PoolStats getStats(final HttpRoute route) {
+ return this.pool.getStats(route);
+ }
+
+}
+
Propchange:
jmeter/trunk/src/protocol/http/org/apache/http/impl/conn/JMeterPoolingClientConnectionManager.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified:
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/control/CacheManager.java
URL:
http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/control/CacheManager.java?rev=1734228&r1=1734227&r2=1734228&view=diff
==============================================================================
---
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/control/CacheManager.java
(original)
+++
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/control/CacheManager.java
Wed Mar 9 12:23:31 2016
@@ -16,8 +16,6 @@
*
*/
-// For unit tests @see TestCookieManager
-
package org.apache.jmeter.protocol.http.control;
import java.io.Serializable;
@@ -77,6 +75,11 @@ public class CacheManager extends Config
private static final int DEFAULT_MAX_SIZE = 5000;
private static final long ONE_YEAR_MS = 365*24*60*60*1000L;
+
+ /** used to share the cache between 2 cache managers
+ * @see CacheManager#createCacheManagerProxy()
+ * @since 3.0 */
+ private transient Map<String, CacheEntry> localCache;
public CacheManager() {
setProperty(new BooleanProperty(CLEAR, false));
@@ -84,6 +87,11 @@ public class CacheManager extends Config
clearCache();
useExpires = false;
}
+
+ CacheManager(Map<String, CacheEntry> localCache, boolean useExpires) {
+ this.localCache = localCache;
+ this.useExpires = useExpires;
+ }
/*
* Holder for storing cache details.
@@ -225,7 +233,7 @@ public class CacheManager extends Config
}
}
// if no-cache is present, ensure that expiresDate remains null,
which forces revalidation
- if(cacheControl != null && !cacheControl.contains("no-cache")) {
+ if(cacheControl != null && !cacheControl.contains("no-cache")) {
// the max-age directive overrides the Expires header,
if(cacheControl.contains(MAX_AGE)) {
long maxAgeInSecs = Long.parseLong(
@@ -255,11 +263,11 @@ public class CacheManager extends Config
+ date);
}
// TODO Can't see anything in SPEC
- expiresDate = new
Date(System.currentTimeMillis()+ONE_YEAR_MS);
+ expiresDate = new
Date(System.currentTimeMillis()+ONE_YEAR_MS);
}
} else {
// TODO Can't see anything in SPEC
- expiresDate = new
Date(System.currentTimeMillis()+ONE_YEAR_MS);
+ expiresDate = new
Date(System.currentTimeMillis()+ONE_YEAR_MS);
}
}
// else expiresDate computed in (expires!=null) condition is
used
@@ -418,8 +426,8 @@ public class CacheManager extends Config
return false;
}
- private Map<String, CacheEntry> getCache(){
- return threadCache.get();
+ private Map<String, CacheEntry> getCache() {
+ return localCache != null?localCache:threadCache.get();
}
public boolean getClearEachIteration() {
@@ -471,6 +479,17 @@ public class CacheManager extends Config
}
};
}
+
+ /**
+ * create a cache manager that share the underlying cache of the current
one
+ * it allows to use the same cache in different threads which does not
inherit from each other
+ * @return a cache manager that share the underlying cache of the current
one
+ * @since 3.0
+ */
+ public CacheManager createCacheManagerProxy() {
+ CacheManager cm = new CacheManager(getCache(), this.useExpires);
+ return cm;
+ }
@Override
public void testStarted() {
@@ -493,7 +512,7 @@ public class CacheManager extends Config
if (getClearEachIteration()) {
clearCache();
}
- useExpires=getUseExpires(); // cache the value
+ useExpires = getUseExpires(); // cache the value
}
}
Modified:
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java
URL:
http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java?rev=1734228&r1=1734227&r2=1734228&view=diff
==============================================================================
---
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java
(original)
+++
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java
Wed Mar 9 12:23:31 2016
@@ -140,6 +140,10 @@ public class HTTPHC4Impl extends HTTPHCA
/** Idle timeout to be applied to connections if no Keep-Alive header is
sent by the server (default 0 = disable) */
private static final int IDLE_TIMEOUT =
JMeterUtils.getPropDefault("httpclient4.idletimeout", 0);
+
+ private static final int VALIDITY_AFTER_INACTIVITY_TIMEOUT =
JMeterUtils.getPropDefault("httpclient4.validate_after_inactivity", 2000);
+
+ private static final int TIME_TO_LIVE =
JMeterUtils.getPropDefault("httpclient4.time_to_live", 2000);
private static final String CONTEXT_METRICS = "jmeter_metrics"; // TODO
hack for metrics related to HTTPCLIENT-1081, to be removed later
@@ -204,6 +208,8 @@ public class HTTPHC4Impl extends HTTPHCA
private static final String USER_TOKEN = "__jmeter.USER_TOKEN__";
//$NON-NLS-1$
static final String SAMPLER_RESULT_TOKEN = "__jmeter.SAMPLER_RESULT__";
//$NON-NLS-1$
+
+ private static final String HTTPCLIENT_TOKEN =
"__jmeter.HTTPCLIENT_TOKEN__";
static {
log.info("HTTP request retry count = "+RETRY_COUNT);
@@ -420,6 +426,7 @@ public class HTTPHC4Impl extends HTTPHCA
return res;
} finally {
currentRequest = null;
+
JMeterContextService.getContext().getSamplerContext().remove(HTTPCLIENT_TOKEN);
}
return res;
}
@@ -686,7 +693,14 @@ public class HTTPHC4Impl extends HTTPHCA
// Lookup key - must agree with all the values used to create the
HttpClient.
HttpClientKey key = new HttpClientKey(url, useProxy, proxyHost,
proxyPort, proxyUser, proxyPass);
- HttpClient httpClient = mapHttpClientPerHttpClientKey.get(key);
+ HttpClient httpClient = null;
+ if(this.testElement.isConcurrentDwn()) {
+ httpClient = (HttpClient)
JMeterContextService.getContext().getSamplerContext().get(HTTPCLIENT_TOKEN);
+ }
+
+ if (httpClient == null) {
+ httpClient = mapHttpClientPerHttpClientKey.get(key);
+ }
if (httpClient != null && resetSSLContext &&
HTTPConstants.PROTOCOL_HTTPS.equalsIgnoreCase(url.getProtocol())) {
((AbstractHttpClient) httpClient).clearRequestInterceptors();
@@ -706,7 +720,11 @@ public class HTTPHC4Impl extends HTTPHCA
if (resolver == null) {
resolver = SystemDefaultDnsResolver.INSTANCE;
}
- MeasuringConnectionManager connManager = new
MeasuringConnectionManager(createSchemeRegistry(), resolver);
+ MeasuringConnectionManager connManager = new
MeasuringConnectionManager(
+ createSchemeRegistry(),
+ resolver,
+ TIME_TO_LIVE,
+ VALIDITY_AFTER_INACTIVITY_TIMEOUT);
// Modern browsers use more connections per host than the current
httpclient default (2)
// when using parallel download the httpclient and connection
manager are shared by the downloads threads
@@ -770,6 +788,10 @@ public class HTTPHC4Impl extends HTTPHCA
}
}
+ if(this.testElement.isConcurrentDwn()) {
+
JMeterContextService.getContext().getSamplerContext().put(HTTPCLIENT_TOKEN,
httpClient);
+ }
+
// TODO - should this be done when the client is created?
// If so, then the details need to be added as part of HttpClientKey
setConnectionAuthorization(httpClient, url, getAuthManager(), key);
Modified:
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java
URL:
http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java?rev=1734228&r1=1734227&r2=1734228&view=diff
==============================================================================
---
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java
(original)
+++
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/HTTPSamplerBase.java
Wed Mar 9 12:23:31 2016
@@ -38,11 +38,6 @@ import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
@@ -59,6 +54,7 @@ import org.apache.jmeter.protocol.http.c
import org.apache.jmeter.protocol.http.parser.BaseParser;
import org.apache.jmeter.protocol.http.parser.LinkExtractorParseException;
import org.apache.jmeter.protocol.http.parser.LinkExtractorParser;
+import
org.apache.jmeter.protocol.http.sampler.ResourcesDownloader.AsynSamplerResultHolder;
import org.apache.jmeter.protocol.http.util.ConversionUtils;
import org.apache.jmeter.protocol.http.util.EncoderCache;
import org.apache.jmeter.protocol.http.util.HTTPArgument;
@@ -191,11 +187,6 @@ public abstract class HTTPSamplerBase ex
public static final boolean BROWSER_COMPATIBLE_MULTIPART_MODE_DEFAULT =
false; // The default setting to be used (i.e. historic)
- private static final long KEEPALIVETIME = 0; // for Thread Pool for
resources but no need to use a special value?
-
- private static final long AWAIT_TERMINATION_TIMEOUT =
- JMeterUtils.getPropDefault("httpsampler.await_termination_timeout",
60); // $NON-NLS-1$ // default value: 60 secs
-
private static final boolean IGNORE_FAILED_EMBEDDED_RESOURCES =
JMeterUtils.getPropDefault("httpsampler.ignore_failed_embedded_resources",
false); // $NON-NLS-1$ // default value: false
@@ -1237,7 +1228,6 @@ public abstract class HTTPSamplerBase ex
int maxConcurrentDownloads = CONCURRENT_POOL_SIZE; // init with
default value
boolean isConcurrentDwn = isConcurrentDwn();
if(isConcurrentDwn) {
-
try {
maxConcurrentDownloads =
Integer.parseInt(getConcurrentPool());
} catch (NumberFormatException nfe) {
@@ -1261,7 +1251,7 @@ public abstract class HTTPSamplerBase ex
log.warn("Null URL detected (should not happen)");
} else {
String urlstr = url.toString();
- String
urlStrEnc=escapeIllegalURLCharacters(encodeSpaces(urlstr));
+ String urlStrEnc =
escapeIllegalURLCharacters(encodeSpaces(urlstr));
if (!urlstr.equals(urlStrEnc)){// There were some
spaces in the URL
try {
url = new URL(urlStrEnc);
@@ -1282,6 +1272,7 @@ public abstract class HTTPSamplerBase ex
setParentSampleSuccess(res, false);
continue;
}
+
if (isConcurrentDwn) {
// if concurrent download emb. resources, add to a
list for async gets later
list.add(new ASyncSample(url, HTTPConstants.GET,
false, frameDepth + 1, getCookieManager(), this));
@@ -1302,67 +1293,31 @@ public abstract class HTTPSamplerBase ex
// IF for download concurrent embedded resources
if (isConcurrentDwn && !list.isEmpty()) {
- final String parentThreadName =
Thread.currentThread().getName();
- // Thread pool Executor to get resources
- // use a LinkedBlockingQueue, note: max pool size doesn't
effect
- final ThreadPoolExecutor exec = new ThreadPoolExecutor(
- maxConcurrentDownloads, maxConcurrentDownloads,
KEEPALIVETIME, TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(),
- new ThreadFactory() {
- @Override
- public Thread newThread(final Runnable r) {
- Thread t = new CleanerThread(new Runnable() {
- @Override
- public void run() {
- try {
- r.run();
- } finally {
-
((CleanerThread)Thread.currentThread()).notifyThreadEnd();
- }
- }
- });
- t.setName(parentThreadName+"-ResDownload-" +
t.getName()); //$NON-NLS-1$
- t.setDaemon(true);
- return t;
- }
- });
+ ResourcesDownloader resourcesDownloader =
ResourcesDownloader.getInstance();
- boolean tasksCompleted = false;
try {
- // sample all resources with threadpool
- final List<Future<AsynSamplerResultHolder>> retExec =
exec.invokeAll(list);
- // call normal shutdown (wait ending all tasks)
- exec.shutdown();
- // put a timeout if tasks couldn't terminate
- exec.awaitTermination(AWAIT_TERMINATION_TIMEOUT,
TimeUnit.SECONDS);
+ // sample all resources
+ final List<Future<AsynSamplerResultHolder>> retExec =
resourcesDownloader.invokeAllAndAwaitTermination(maxConcurrentDownloads, list);
CookieManager cookieManager = getCookieManager();
// add result to main sampleResult
for (Future<AsynSamplerResultHolder> future : retExec) {
- AsynSamplerResultHolder binRes;
- try {
- binRes = future.get(1, TimeUnit.MILLISECONDS);
- if(cookieManager != null) {
- CollectionProperty cookies =
binRes.getCookies();
- for (JMeterProperty jMeterProperty : cookies) {
- Cookie cookie = (Cookie)
jMeterProperty.getObjectValue();
- cookieManager.add(cookie);
- }
+ // this call will not block as the futures return by
invokeAllAndAwaitTermination
+ // are either done or cancelled
+ AsynSamplerResultHolder binRes = future.get();
+ if(cookieManager != null) {
+ CollectionProperty cookies = binRes.getCookies();
+ for (JMeterProperty jMeterProperty : cookies) {
+ Cookie cookie = (Cookie)
jMeterProperty.getObjectValue();
+ cookieManager.add(cookie);
}
- res.addSubResult(binRes.getResult());
- setParentSampleSuccess(res, res.isSuccessful() &&
(binRes.getResult() != null ? binRes.getResult().isSuccessful():true));
- } catch (TimeoutException e) {
- errorResult(e, res);
}
+ res.addSubResult(binRes.getResult());
+ setParentSampleSuccess(res, res.isSuccessful() &&
(binRes.getResult() != null ? binRes.getResult().isSuccessful():true));
}
- tasksCompleted = exec.awaitTermination(1,
TimeUnit.MILLISECONDS); // did all the tasks finish?
} catch (InterruptedException ie) {
log.warn("Interrupted fetching embedded resources", ie);
// $NON-NLS-1$
} catch (ExecutionException ee) {
log.warn("Execution issue when fetching embedded
resources", ee); // $NON-NLS-1$
- } finally {
- if (!tasksCompleted) {
- exec.shutdownNow(); // kill any remaining tasks
- }
}
}
}
@@ -1972,7 +1927,7 @@ public abstract class HTTPSamplerBase ex
// We don't want to use CacheManager clone but the parent one, and
CacheManager is Thread Safe
CacheManager cacheManager = base.getCacheManager();
if (cacheManager != null) {
- this.sampler.setCacheManagerProperty(cacheManager);
+
this.sampler.setCacheManagerProperty(cacheManager.createCacheManagerProxy());
}
if(cookieManager != null) {
@@ -1996,67 +1951,6 @@ public abstract class HTTPSamplerBase ex
}
}
- /**
- * Custom thread implementation that allows notification of threadEnd
- *
- */
- private static class CleanerThread extends Thread {
- private final List<HTTPSamplerBase> samplersToNotify = new
ArrayList<>();
- /**
- * @param runnable Runnable
- */
- public CleanerThread(Runnable runnable) {
- super(runnable);
- }
-
- /**
- * Notify of thread end
- */
- public void notifyThreadEnd() {
- for (HTTPSamplerBase samplerBase : samplersToNotify) {
- samplerBase.threadFinished();
- }
- samplersToNotify.clear();
- }
-
- /**
- * Register sampler to be notify at end of thread
- * @param sampler {@link HTTPSamplerBase}
- */
- public void registerSamplerForEndNotification(HTTPSamplerBase sampler)
{
- this.samplersToNotify.add(sampler);
- }
- }
-
- /**
- * Holder of AsynSampler result
- */
- private static class AsynSamplerResultHolder {
- private final HTTPSampleResult result;
- private final CollectionProperty cookies;
- /**
- * @param result {@link HTTPSampleResult} to hold
- * @param cookies cookies to hold
- */
- public AsynSamplerResultHolder(HTTPSampleResult result,
CollectionProperty cookies) {
- super();
- this.result = result;
- this.cookies = cookies;
- }
- /**
- * @return the result
- */
- public HTTPSampleResult getResult() {
- return result;
- }
- /**
- * @return the cookies
- */
- public CollectionProperty getCookies() {
- return cookies;
- }
- }
-
/**
* @see
org.apache.jmeter.samplers.AbstractSampler#applies(org.apache.jmeter.config.ConfigTestElement)
*/
Modified:
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/MeasuringConnectionManager.java
URL:
http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/MeasuringConnectionManager.java?rev=1734228&r1=1734227&r2=1734228&view=diff
==============================================================================
---
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/MeasuringConnectionManager.java
(original)
+++
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/MeasuringConnectionManager.java
Wed Mar 9 12:23:31 2016
@@ -38,6 +38,7 @@ import org.apache.http.conn.DnsResolver;
import org.apache.http.conn.ManagedClientConnection;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.SchemeRegistry;
+import org.apache.http.impl.conn.JMeterPoolingClientConnectionManager;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.HttpContext;
@@ -48,10 +49,14 @@ import org.apache.jmeter.samplers.Sample
* that wraps all connection requests into time-measured implementation a
private
* MeasuringConnectionRequest
*/
-public class MeasuringConnectionManager extends PoolingClientConnectionManager
{
+public class MeasuringConnectionManager extends
JMeterPoolingClientConnectionManager {
- public MeasuringConnectionManager(SchemeRegistry schemeRegistry,
DnsResolver resolver) {
- super(schemeRegistry, resolver);
+
+ public MeasuringConnectionManager(SchemeRegistry schemeRegistry,
+ DnsResolver resolver,
+ int timeToLiveMs,
+ int validateAfterInactivityMs) {
+ super(schemeRegistry, timeToLiveMs, TimeUnit.MILLISECONDS, resolver,
validateAfterInactivityMs);
}
@Override
Added:
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/ResourcesDownloader.java
URL:
http://svn.apache.org/viewvc/jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/ResourcesDownloader.java?rev=1734228&view=auto
==============================================================================
---
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/ResourcesDownloader.java
(added)
+++
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/ResourcesDownloader.java
Wed Mar 9 12:23:31 2016
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.jmeter.protocol.http.sampler;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.jmeter.testelement.property.CollectionProperty;
+import org.apache.jmeter.util.JMeterUtils;
+import org.apache.jorphan.logging.LoggingManager;
+import org.apache.log.Logger;
+
+/**
+ * Manages the parallel http resources download.<br>
+ * A shared thread pool is used by all the sample.<br>
+ * A sampler will usually do the following
+ * <pre> {@code
+ * // list of AsynSamplerResultHolder to download
+ * List<Callable<AsynSamplerResultHolder>> list = ...
+ *
+ * // max parallel downloads
+ * int maxConcurrentDownloads = ...
+ *
+ * // get the singleton instance
+ * ResourcesDownloader resourcesDownloader =
ResourcesDownloader.getInstance();
+ *
+ * // schedule the downloads and wait for the completion
+ * List<Future<AsynSamplerResultHolder>> retExec =
resourcesDownloader.invokeAllAndAwaitTermination(maxConcurrentDownloads, list);
+ *
+ * }</pre>
+ *
+ * the call to invokeAllAndAwaitTermination will block until the downloads
complete or get interrupted<br>
+ * the Future list only contains task that have been scheduled in the
threadpool.<br>
+ * The status of those futures are either done or cancelled<br>
+ * <br>
+ *
+ * Future enhancements :
+ * <ul>
+ * <li>this implementation should be replaced with a NIO async download
+ * in order to reduce the number of threads needed</li>
+ * </ul>
+ * @since 3.0
+ */
+public class ResourcesDownloader {
+
+ private static final Logger LOG = LoggingManager.getLoggerForClass();
+
+ /** this is the maximum time that excess idle threads will wait for new
tasks before terminating */
+ private static final long THREAD_KEEP_ALIVE_TIME =
JMeterUtils.getPropDefault("httpsampler.parallel_download_thread_keepalive_inseconds",
60L);
+
+ private static final int MIN_POOL_SIZE = 1;
+ private static final int MAX_POOL_SIZE = Integer.MAX_VALUE;
+
+ private static final ResourcesDownloader INSTANCE = new
ResourcesDownloader();
+
+ public static ResourcesDownloader getInstance() {
+ return INSTANCE;
+ }
+
+
+ private ThreadPoolExecutor concurrentExecutor = null;
+
+ private ResourcesDownloader() {
+ init();
+ }
+
+
+ private void init() {
+ LOG.info("Creating ResourcesDownloader with
keepalive_inseconds:"+THREAD_KEEP_ALIVE_TIME);
+ ThreadPoolExecutor exec = new ThreadPoolExecutor(
+ MIN_POOL_SIZE, MAX_POOL_SIZE, THREAD_KEEP_ALIVE_TIME,
TimeUnit.SECONDS,
+ new SynchronousQueue<Runnable>(),
+ new ThreadFactory() {
+ @Override
+ public Thread newThread(final Runnable r) {
+ Thread t = new Thread(r);
+ t.setName("ResDownload-" + t.getName()); //$NON-NLS-1$
+ t.setDaemon(true);
+ return t;
+ }
+ }) {
+
+ };
+ concurrentExecutor = exec;
+ }
+
+ /**
+ * this method will try to shrink the thread pool size as much as possible
+ * it should be called at the end of a test
+ */
+ public void shrink() {
+ if(concurrentExecutor.getPoolSize() > MIN_POOL_SIZE) {
+ // drain the queue
+ concurrentExecutor.purge();
+ List<Runnable> drainList = new ArrayList<>();
+ concurrentExecutor.getQueue().drainTo(drainList);
+ if(!drainList.isEmpty()) {
+ LOG.warn("the pool executor workqueue is not empty size=" +
drainList.size());
+ for (Runnable runnable : drainList) {
+ if(runnable instanceof Future<?>) {
+ Future<?> f = (Future<?>) runnable;
+ f.cancel(true);
+ }
+ else {
+ LOG.warn("Content of workqueue is not an instance of
Future");
+ }
+ }
+ }
+
+ // this will force the release of the extra threads that are idle
+ // the remaining extra threads will be released with the
keepAliveTime of the thread
+ concurrentExecutor.setMaximumPoolSize(MIN_POOL_SIZE);
+
+ // do not immediately restore the MaximumPoolSize as it will block
the release of the threads
+ }
+ }
+
+ // probablyTheBestMethodNameInTheUniverseYeah!
+ /**
+ * This method will block until the downloads complete or it get
interrupted
+ * the Future list returned by this method only contains tasks that have
been scheduled in the threadpool.<br>
+ * The status of those futures are either done or cancelled
+ *
+ * @param maxConcurrentDownloads max concurrent downloads
+ * @param list list of resources to download
+ * @return list tasks that have been scheduled
+ * @throws InterruptedException
+ */
+ public List<Future<AsynSamplerResultHolder>>
invokeAllAndAwaitTermination(int maxConcurrentDownloads,
List<Callable<AsynSamplerResultHolder>> list) throws InterruptedException {
+ List<Future<AsynSamplerResultHolder>> submittedTasks = new
ArrayList<>();
+
+ // paranoid fast path
+ if(list.isEmpty()) {
+ return submittedTasks;
+ }
+
+ // restore MaximumPoolSize original value
+ concurrentExecutor.setMaximumPoolSize(MAX_POOL_SIZE);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("PoolSize=" + concurrentExecutor.getPoolSize()+"
LargestPoolSize=" + concurrentExecutor.getLargestPoolSize());
+ }
+
+ CompletionService<AsynSamplerResultHolder> completionService = new
ExecutorCompletionService<>(concurrentExecutor);
+ int remainingTasksToTake = list.size();
+
+ try {
+ // push the task in the threadpool until <maxConcurrentDownloads>
is reached
+ int i = 0;
+ for (i = 0; i < Math.min(maxConcurrentDownloads, list.size());
i++) {
+ Callable<AsynSamplerResultHolder> task = list.get(i);
+ submittedTasks.add(completionService.submit(task));
+ }
+
+ // push the remaining tasks but ensure we use at most
<maxConcurrentDownloads> threads
+ // wait for a previous download to finish before submitting a new
one
+ for (; i < list.size(); i++) {
+ Callable<AsynSamplerResultHolder> task = list.get(i);
+ completionService.take();
+ remainingTasksToTake--;
+ submittedTasks.add(completionService.submit(task));
+ }
+
+ // all the resources downloads are in the thread pool queue
+ // wait for the completion of all downloads
+ while (remainingTasksToTake > 0) {
+ completionService.take();
+ remainingTasksToTake--;
+ }
+ }
+ finally {
+ //bug 51925 : Calling Stop on Test leaks executor threads when
concurrent download of resources is on
+ if(remainingTasksToTake > 0) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Interrupted while waiting for resource
downloads : cancelling remaining tasks");
+ }
+ for (Future<AsynSamplerResultHolder> future : submittedTasks) {
+ if(!future.isDone()) {
+ future.cancel(true);
+ }
+ }
+ }
+ }
+
+ return submittedTasks;
+ }
+
+
+ /**
+ * Holder of AsynSampler result
+ */
+ public static class AsynSamplerResultHolder {
+ private final HTTPSampleResult result;
+ private final CollectionProperty cookies;
+
+ /**
+ * @param result {@link HTTPSampleResult} to hold
+ * @param cookies cookies to hold
+ */
+ public AsynSamplerResultHolder(HTTPSampleResult result,
CollectionProperty cookies) {
+ super();
+ this.result = result;
+ this.cookies = cookies;
+ }
+
+ /**
+ * @return the result
+ */
+ public HTTPSampleResult getResult() {
+ return result;
+ }
+
+ /**
+ * @return the cookies
+ */
+ public CollectionProperty getCookies() {
+ return cookies;
+ }
+ }
+
+}
Propchange:
jmeter/trunk/src/protocol/http/org/apache/jmeter/protocol/http/sampler/ResourcesDownloader.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: jmeter/trunk/xdocs/changes.xml
URL:
http://svn.apache.org/viewvc/jmeter/trunk/xdocs/changes.xml?rev=1734228&r1=1734227&r2=1734228&view=diff
==============================================================================
--- jmeter/trunk/xdocs/changes.xml (original)
+++ jmeter/trunk/xdocs/changes.xml Wed Mar 9 12:23:31 2016
@@ -146,6 +146,7 @@ Summary
<li><bug>59129</bug>HTTP Request : Simplify GUI with simple/advanced
Tabs</li>
<li><bug>59033</bug>Parallel Download : Rework Parser classes hierarchy to
allow pluging parsers for different mime types</li>
<li><bug>59146</bug>MeasuringConnectionManager is not Thread Safe (nightly
before 3.0)</li>
+ <li><bug>52073</bug>Embedded Resources Parallel download : Improve
performances by avoiding shutdown of ThreadPoolExecutor at each sample </li>
</ul>
<h3>Other samplers</h3>
