andreaslind01 commented on code in PR #6742: URL: https://github.com/apache/jmeter/pull/6742#discussion_r4183336128
########## src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPHC5Impl.java: ########## @@ -0,0 +1,1867 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jmeter.protocol.http.sampler; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.security.Principal; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.WeakHashMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import javax.net.ssl.SSLContext; +import javax.security.auth.Subject; + +import org.apache.hc.client5.http.DnsResolver; +import org.apache.hc.client5.http.HttpRoute; +import org.apache.hc.client5.http.RouteInfo; +import org.apache.hc.client5.http.SystemDefaultDnsResolver; +import org.apache.hc.client5.http.async.methods.AbstractBinResponseConsumer; +import org.apache.hc.client5.http.auth.AuthSchemeFactory; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.Credentials; +import org.apache.hc.client5.http.auth.CredentialsProvider; +import org.apache.hc.client5.http.auth.KerberosConfig; +import org.apache.hc.client5.http.auth.KerberosCredentials; +import org.apache.hc.client5.http.auth.StandardAuthScheme; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.classic.ExecChainHandler; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.config.TlsConfig; +import org.apache.hc.client5.http.entity.DecompressingEntity; +import org.apache.hc.client5.http.entity.DeflateInputStreamFactory; +import org.apache.hc.client5.http.entity.GZIPInputStreamFactory; +import org.apache.hc.client5.http.entity.InputStreamFactory; +import org.apache.hc.client5.http.entity.UrlEncodedFormEntity; +import org.apache.hc.client5.http.entity.mime.FileBody; +import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder; +import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; +import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; +import org.apache.hc.client5.http.impl.async.HttpAsyncClients; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.auth.BasicSchemeFactory; +import org.apache.hc.client5.http.impl.auth.BearerSchemeFactory; +import org.apache.hc.client5.http.impl.auth.DigestSchemeFactory; +import org.apache.hc.client5.http.impl.auth.KerberosSchemeFactory; +import org.apache.hc.client5.http.impl.auth.SPNegoSchemeFactory; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner; +import org.apache.hc.client5.http.io.ConnectionEndpoint; +import org.apache.hc.client5.http.io.HttpClientConnectionManager; +import org.apache.hc.client5.http.io.LeaseRequest; +import org.apache.hc.client5.http.nio.AsyncClientConnectionManager; +import org.apache.hc.client5.http.nio.AsyncConnectionEndpoint; +import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.client5.http.protocol.RedirectLocations; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.HostnameVerificationPolicy; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.client5.http.ssl.TlsSocketStrategy; +import org.apache.hc.client5.http.utils.URIUtils; +import org.apache.hc.core5.concurrent.FutureCallback; +import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HeaderElement; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.HttpRequestInterceptor; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.HttpVersion; +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.http.config.Lookup; +import org.apache.hc.core5.http.config.RegistryBuilder; +import org.apache.hc.core5.http.io.entity.ByteArrayEntity; +import org.apache.hc.core5.http.io.entity.FileEntity; +import org.apache.hc.core5.http.io.entity.HttpEntityWrapper; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.http.message.BasicClassicHttpResponse; +import org.apache.hc.core5.http.message.BasicHeaderValueParser; +import org.apache.hc.core5.http.message.BasicNameValuePair; +import org.apache.hc.core5.http.message.ParserCursor; +import org.apache.hc.core5.http.nio.entity.BasicAsyncEntityProducer; +import org.apache.hc.core5.http.nio.entity.FileEntityProducer; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.http.nio.support.BasicRequestProducer; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.apache.hc.core5.http2.HttpVersionPolicy; +import org.apache.hc.core5.http2.config.H2Config; +import org.apache.hc.core5.io.CloseMode; +import org.apache.hc.core5.net.URIAuthority; +import org.apache.hc.core5.pool.PoolConcurrencyPolicy; +import org.apache.hc.core5.reactor.ConnectionInitiator; +import org.apache.hc.core5.util.TimeValue; +import org.apache.hc.core5.util.Timeout; +import org.apache.hc.core5.util.VersionInfo; +import org.apache.jmeter.protocol.http.control.AuthManager; +import org.apache.jmeter.protocol.http.control.Authorization; +import org.apache.jmeter.protocol.http.control.CacheManager; +import org.apache.jmeter.protocol.http.control.CookieManager; +import org.apache.jmeter.protocol.http.control.DNSCacheManager; +import org.apache.jmeter.protocol.http.control.HeaderManager; +import org.apache.jmeter.protocol.http.util.HTTPArgument; +import org.apache.jmeter.protocol.http.util.HTTPConstants; +import org.apache.jmeter.protocol.http.util.HTTPFileArg; +import org.apache.jmeter.samplers.SampleResult; +import org.apache.jmeter.services.FileServer; +import org.apache.jmeter.testelement.property.JMeterProperty; +import org.apache.jmeter.threads.JMeterContext; +import org.apache.jmeter.threads.JMeterContextService; +import org.apache.jmeter.threads.JMeterVariables; +import org.apache.jmeter.util.JMeterUtils; +import org.apache.jmeter.util.JsseSSLManager; +import org.apache.jmeter.util.SSLManager; +import org.apache.jorphan.io.CountingInputStream; +import org.apache.jorphan.io.CountingOutputStream; +import org.apache.jorphan.util.JOrphanUtils; +import org.apache.jorphan.util.StringUtilities; +import org.brotli.dec.BrotliInputStream; +import org.ietf.jgss.GSSCredential; +import org.ietf.jgss.GSSException; +import org.ietf.jgss.GSSManager; +import org.ietf.jgss.Oid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * HTTP Sampler using Apache HttpClient 5.x. + */ +public class HTTPHC5Impl extends HTTPHCAbstractImpl { + + private static final Logger log = LoggerFactory.getLogger(HTTPHC5Impl.class); + + /** Key used to store the current {@link SampleResult} in the {@link HttpClientContext}. */ + static final String CONTEXT_ATTRIBUTE_SAMPLER_RESULT = "__jmeter.S_R__"; //$NON-NLS-1$ + private static final String CONTEXT_ATTRIBUTE_RESPONSE_LATENCY = "__jmeter.S_R_LATENCY__"; //$NON-NLS-1$ + private static final String CONTEXT_ATTRIBUTE_RESPONSE_BODY_SIZE = "__jmeter.S_R_BODY_SIZE__"; //$NON-NLS-1$ + private static final String CONTEXT_ATTRIBUTE_RESPONSE_BODY_COUNTER = "__jmeter.S_R_BODY_COUNTER__"; //$NON-NLS-1$ + + /** + * Clients of every JMeter thread, looked up by the {@link JMeterContext} of that thread instead + * of by the thread itself. The threads that download embedded resources in parallel are pooled + * and shared by all JMeter threads, and they adopt the context of the JMeter thread they are + * working for, so looking the clients up by context hands them the clients of that thread. That + * shares a connection between a sample and its embedded resources, which lets HTTP/2 multiplex + * them, and it keeps a JMeter thread from closing clients another one is still using. + * + * <p>The keys are weak, so that a context whose clients are never closed explicitly, like the + * one of a thread of the HTTP(S) Test Script Recorder, does not keep the entry alive forever. + */ + private static final Map<JMeterContext, Map<HttpClientKey, CloseableHttpClient>> HTTP_CLIENTS = + Collections.synchronizedMap(new WeakHashMap<>()); + + /** + * HTTP/2 clients of every JMeter thread, see {@link #HTTP_CLIENTS} for how they are shared with + * the threads that download embedded resources in parallel. + */ + private static final Map<JMeterContext, Map<HttpClientKey, CloseableHttpAsyncClient>> HTTP_2_CLIENTS = + Collections.synchronizedMap(new WeakHashMap<>()); + + /** Multiplex concurrent message exchanges over a single HTTP/2 connection. */ + private static final boolean HTTP_2_MULTIPLEXING = + JMeterUtils.getPropDefault("httpclient5.h2.multiplexing", true); + + /** + * HTTP/2 flow control window announced per stream when the {@code httpclient5.h2.initial_window_size} + * property does not say otherwise. It caps how many bytes the server may send before it has to wait + * for a window update, so the throughput of a single response can never exceed this window divided by + * the round trip time. The default of HttpClient is the 64 kB the HTTP/2 specification mandates, which + * throttles a download over a link with any noticeable latency to a fraction of the available + * bandwidth. JMeter therefore announces the same 16 MB the JDK client uses by default. The window + * is only a promise to the server, the body is consumed as it arrives and not buffered up to this + * size. + */ + static final int DEFAULT_HTTP_2_INITIAL_WINDOW_SIZE = 16 * 1024 * 1024; + + /** + * Use HTTP/2 without TLS (h2c with prior knowledge) when HTTP/2 is selected for a {@code http://} URL. + * Without prior knowledge such requests silently fall back to HTTP/1.1, as ALPN is unavailable. + */ + private static final boolean HTTP_2_PRIOR_KNOWLEDGE = + JMeterUtils.getPropDefault("httpclient5.h2.prior_knowledge", false); + + private static final int HTTP_2_MAX_CONNECTIONS_PER_ROUTE = + JMeterUtils.getPropDefault("httpclient5.h2.max_connections_per_route", 6); + + /** + * Number of bytes of an HTTP/2 request body that are buffered in memory before the body is + * spilled to a temporary file. Keeping the common small bodies in heap avoids a temporary file + * per sample, while a larger body is streamed from disk so that a big upload is never + * materialized as a byte array. + */ + private static final int HTTP_2_BODY_SPILL_THRESHOLD = + JMeterUtils.getPropDefault("httpclient5.h2.body_spill_threshold", 256 * 1024); + + /** Milliseconds of inactivity after which a pooled connection is re-validated, visible for testing. */ + static long validateAfterInactivityMillis = + JMeterUtils.getPropDefault("httpclient5.validate_after_inactivity", 2000L); + + /** + * Name of the property that suppresses the {@code User-Agent} header HttpClient sends when the + * test plan does not define one itself. + */ + private static final String DISABLE_DEFAULT_UA_PROPERTY = "httpclient5.default_user_agent_disabled"; + + /** + * {@code User-Agent} JMeter adds to requests without one, so it shows up in the sample result and + * is accounted for in the sent bytes, instead of being added invisibly by HttpClient. + */ + private static final String DEFAULT_USER_AGENT = VersionInfo.getSoftwareInfo("Apache-HttpClient", + "org.apache.hc.client5", org.apache.hc.client5.http.impl.classic.HttpClientBuilder.class); + + /** Remembers whether the request had a {@code User-Agent} header before HttpClient added its own. */ + private static final String CONTEXT_ATTRIBUTE_USER_AGENT_PRESENT = "__jmeter.U_A__"; + + /** Shown instead of the request body when the entity cannot be read a second time. */ + private static final String NON_REPEATABLE_ENTITY_PREVIEW = + "<Entity was not repeatable, cannot view what was sent>"; // $NON-NLS-1$ + + private static final HttpRequestInterceptor RECORD_USER_AGENT_PRESENCE = (request, entity, context) -> + context.setAttribute(CONTEXT_ATTRIBUTE_USER_AGENT_PRESENT, request.containsHeader(HttpHeaders.USER_AGENT)); + + private static final HttpRequestInterceptor REMOVE_DEFAULT_USER_AGENT = (request, entity, context) -> { + if (!Boolean.TRUE.equals(context.getAttribute(CONTEXT_ATTRIBUTE_USER_AGENT_PRESENT))) { + request.removeHeaders(HttpHeaders.USER_AGENT); + } + }; + + private static final H2Config HTTP_2_CONFIG = createHttp2Config(); + + /** + * Request that the Kerberos credentials of the JMeter user are delegated to the server, so + * that it can act on behalf of the user, see the {@code kerberos.spnego.delegate_cred} + * property. + */ + private static final boolean KERBEROS_DELEGATE_CRED = + JMeterUtils.getPropDefault("kerberos.spnego.delegate_cred", false); + + /** + * HttpClient 5 only offers {@code Bearer}, {@code Digest} and {@code Basic} by default, so the + * negotiation based schemes have to be requested explicitly for a Kerberos authorization. + */ + @SuppressWarnings("deprecation") // The GSS based auth schemes of HttpClient 5 have no replacement yet + private static final List<String> KERBEROS_PREFERRED_AUTH_SCHEMES = + List.of(StandardAuthScheme.SPNEGO, StandardAuthScheme.KERBEROS); + + /** + * Auth schemes offered to a proxy that can be authenticated with Kerberos. Unlike the target + * schemes these keep the password based schemes as a fallback, so that a proxy which offers + * {@code Negotiate} next to {@code Basic} or {@code Digest} can still be used. + */ + @SuppressWarnings("deprecation") // The GSS based auth schemes of HttpClient 5 have no replacement yet + private static final List<String> KERBEROS_PROXY_PREFERRED_AUTH_SCHEMES = + List.of(StandardAuthScheme.SPNEGO, StandardAuthScheme.KERBEROS, StandardAuthScheme.BEARER, + StandardAuthScheme.DIGEST, StandardAuthScheme.BASIC); + + private static final Oid SPNEGO_OID = createOid("1.3.6.1.5.5.2"); + + private static final Oid KERBEROS_OID = createOid("1.2.840.113554.1.2.2"); + + /** + * Credentials without a principal, telling the negotiation based auth schemes to use the + * credentials of the JAAS {@link Subject} the request is executed with. + */ + private static final Credentials USE_JAAS_CREDENTIALS = new Credentials() { + @Override + public Principal getUserPrincipal() { + return null; + } + + @Override + @SuppressWarnings("deprecation") // Credentials.getPassword is deprecated, but has to be implemented + public char[] getPassword() { + return new char[0]; + } + }; + + private static final String[] HEADERS_TO_SAVE = {HttpHeaders.CONTENT_LENGTH, HttpHeaders.CONTENT_ENCODING, + HttpHeaders.CONTENT_MD5}; + + private static final String[] SOCKET_PROTOCOL_ARRAY = + JMeterUtils.getArrayPropDefault("https.socket.protocols", null); + + private static final String[] SOCKET_CIPHER_ARRAY = + JMeterUtils.getArrayPropDefault("https.socket.ciphers", null); + + private static final String[] CIPHER_SUITE_ARRAY = + JMeterUtils.getArrayPropDefault("https.cipherSuites", SOCKET_CIPHER_ARRAY); + + // HttpClient 5.6 switched BrotliInputStreamFactory to the optional brotli4j library, so decode "br" + // with the org.brotli:dec library JMeter ships, like HTTPHC4Impl does. + @SuppressWarnings("deprecation") + private static final InputStreamFactory BROTLI = BrotliInputStream::new; + + // The InputStreamFactory-based decoders are deprecated in favour of the @Internal ContentCodecRegistry, + // so keep using them until a public replacement is available. + @SuppressWarnings("deprecation") + private static final Lookup<InputStreamFactory> CONTENT_DECODERS = RegistryBuilder.<InputStreamFactory>create() + .register("br", BROTLI) + .register("gzip", GZIPInputStreamFactory.getInstance()) + .register("x-gzip", GZIPInputStreamFactory.getInstance()) + .register("deflate", DeflateInputStreamFactory.getInstance()) + .build(); + + private static final ExecChainHandler RESPONSE_CONTENT_ENCODING = (request, scope, chain) -> { + HttpClientContext context = scope.clientContext; + RequestConfig requestConfig = context.getRequestConfig(); + if (requestConfig == null) { + requestConfig = RequestConfig.DEFAULT; + } + ClassicHttpResponse response = chain.proceed(request, scope); + if (!requestConfig.isContentCompressionEnabled()) { + return response; + } + return decompressResponse(response, context); + }; + + /** + * Decodes the response body while keeping the {@code Content-Encoding}, {@code Content-Length} and + * {@code Content-MD5} headers, so the sample result still reports what the server actually sent. + * JMeter therefore disables the transparent decompression of HttpClient, which drops those headers, + * and decodes the response itself for both the HTTP/1.1 and the HTTP/2 transport. + */ + @SuppressWarnings("deprecation") // DecompressingEntity is superseded by the @Internal ContentCodecRegistry + private static ClassicHttpResponse decompressResponse(ClassicHttpResponse response, HttpContext context) { + HttpEntity entity = response.getEntity(); + if (entity == null || entity.getContentLength() == 0) { + return response; + } + + String contentEncoding = entity.getContentEncoding(); + if (contentEncoding == null) { + Header encodingHeader = response.getFirstHeader(HttpHeaders.CONTENT_ENCODING); + if (encodingHeader != null) { + contentEncoding = encodingHeader.getValue(); + } + } + if (contentEncoding == null) { + return response; + } + + Header[][] headersToSave = new Header[HEADERS_TO_SAVE.length][]; + for (int i = 0; i < HEADERS_TO_SAVE.length; i++) { + headersToSave[i] = response.getHeaders(HEADERS_TO_SAVE[i]); + } + HeaderElement[] codecs = BasicHeaderValueParser.INSTANCE.parseElements(contentEncoding, + new ParserCursor(0, contentEncoding.length())); + CountingEntity countingEntity = new CountingEntity(entity); + boolean decompressing = false; + for (HeaderElement codec : codecs) { + InputStreamFactory decoderFactory = CONTENT_DECODERS.lookup(codec.getName().toLowerCase(Locale.ROOT)); + if (decoderFactory != null) { + if (!decompressing) { + response.setEntity(countingEntity); + context.setAttribute(CONTEXT_ATTRIBUTE_RESPONSE_BODY_COUNTER, countingEntity); + decompressing = true; + } + response.setEntity(new DecompressingEntity(response.getEntity(), decoderFactory)); + response.removeHeaders(HttpHeaders.CONTENT_LENGTH); + response.removeHeaders(HttpHeaders.CONTENT_ENCODING); + response.removeHeaders(HttpHeaders.CONTENT_MD5); + } + } + for (Header[] headers : headersToSave) { + for (Header header : headers) { + if (!response.containsHeader(header.getName())) { + response.addHeader(header); + } + } + } + return response; + } + + private volatile org.apache.hc.client5.http.classic.methods.HttpUriRequestBase currentRequest; + private volatile Future<?> currentResponseFuture; + private volatile boolean interruptRequested; + + private static ClientTlsStrategyBuilder createTlsStrategyBuilder() { + try { + SSLContext sslContext = ((JsseSSLManager) SSLManager.getInstance()).getContext(); + ClientTlsStrategyBuilder builder = ClientTlsStrategyBuilder.create() + .setSslContext(sslContext) + // Leave hostname verification to the no-op verifier below. Without CLIENT the policy would + // default to BOTH, and the JSSE built-in endpoint identification would reject the + // self-signed certificates JMeter deliberately accepts when testing. + .setHostVerificationPolicy(HostnameVerificationPolicy.CLIENT) + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE); + if (SOCKET_PROTOCOL_ARRAY != null) { + builder.setTlsVersions(SOCKET_PROTOCOL_ARRAY); + } + if (CIPHER_SUITE_ARRAY != null) { + builder.setCiphers(CIPHER_SUITE_ARRAY); + } + return builder; + } catch (GeneralSecurityException e) { + throw new IllegalStateException("Could not create TLS strategy", e); + } + } + + private static TlsSocketStrategy createTlsSocketStrategy() { + return createTlsStrategyBuilder().buildClassic(); + } + + private static TlsStrategy createTlsStrategy() { + return createTlsStrategyBuilder().buildAsync(); Review Comment: This has been addressed and should be resolved by commit 2e83dd3. Thanks for pointing this out. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
