shinseongsu commented on PR #906:
URL:
https://github.com/apache/httpcomponents-client/pull/906#issuecomment-6707902063
Hi @rursprung, thanks for working on this! 🙂 I was curious how the new
meters behave with gzip and HTTP/2, so I tried the PR (74d9bc11c) against a
small local server. I'm sharing the numbers in case they help.
Setup: `HttpClientObservationSupport.enable(...)` with `CONTENT_LENGTH`, and
a 100,000 byte body (133 bytes after gzip).
| Case | Wire | Recorded |
|---|---|---|
| plain request | 100000 | 100000 ✅ |
| request via `EntityBuilder.gzipCompressed()` | 133 (chunked) | nothing |
| pre-gzipped request entity | 133 | 133 |
| gzip response, decompressed by the client | 133 | nothing |
| gzip response, content compression off | 133 | 133 |
| async HTTP/2 response with `Content-Length: 100000` | 100000 | nothing |
So I couldn't find a case where the uncompressed size shows up 😅 When
something is recorded, it's the same as the `Content-Length` on the wire. The
existing `IO` counters gave me the same numbers, since they also read
`getContentLength()`.
For the OpenSearch case I think this means the decoded `_bulk` size isn't
visible once gzip is on. And since opensearch-java compresses the request
itself before the exec chain, I'm not sure the client can see the decoded
request size at all. Counting the bytes that actually go through (like Arturo
mentioned) might be the way to get real numbers.
If you want to try it yourself, drop the test below into
`httpclient5-observation/src/test/java/org/apache/hc/client5/http/observation/interceptors/`
and run:
```
./mvnw -pl httpclient5-observation -am test
-Dtest=ContentLengthSemanticsTest -Dsurefire.failIfNoSpecifiedTests=false
```
All 4 tests pass on the PR branch. The assertions just describe what happens
today.
Hope this is useful! 🙏
<details>
<summary>ContentLengthSemanticsTest.java</summary>
```java
package org.apache.hc.client5.http.observation.interceptors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPOutputStream;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.apache.hc.client5.http.async.methods.SimpleHttpResponse;
import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.entity.EntityBuilder;
import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder;
import org.apache.hc.client5.http.impl.async.HttpAsyncClients;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.observation.HttpClientObservationSupport;
import org.apache.hc.client5.http.observation.MetricConfig;
import org.apache.hc.client5.http.observation.ObservingOptions;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.EntityDetails;
import org.apache.hc.core5.http.HttpException;
import org.apache.hc.core5.http.HttpRequest;
import org.apache.hc.core5.http.Message;
import org.apache.hc.core5.http.URIScheme;
import org.apache.hc.core5.http.impl.bootstrap.HttpAsyncServer;
import org.apache.hc.core5.http.impl.bootstrap.HttpServer;
import org.apache.hc.core5.http.impl.bootstrap.ServerBootstrap;
import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.BasicHttpResponse;
import org.apache.hc.core5.http.nio.AsyncRequestConsumer;
import org.apache.hc.core5.http.nio.AsyncServerRequestHandler;
import org.apache.hc.core5.http.nio.entity.BasicAsyncEntityConsumer;
import org.apache.hc.core5.http.nio.entity.BasicAsyncEntityProducer;
import org.apache.hc.core5.http.nio.support.BasicRequestConsumer;
import org.apache.hc.core5.http.nio.support.BasicResponseProducer;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.http2.HttpVersionPolicy;
import org.apache.hc.core5.http2.impl.nio.bootstrap.H2ServerBootstrap;
import org.apache.hc.core5.io.CloseMode;
import org.apache.hc.core5.reactor.ListenerEndpoint;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/**
* What CONTENT_LENGTH records when compression or HTTP/2 is involved.
* Body: 100_000 bytes of 'a' (gzips to 133 bytes). Assertions describe the
observed behaviour.
*/
class ContentLengthSemanticsTest {
private static final String REQUEST = "t.request.content.length";
private static final String RESPONSE = "t.response.content.length";
private static final byte[] BODY = new byte[100_000];
static {
Arrays.fill(BODY, (byte) 'a');
}
private byte[] gzippedBody;
private HttpServer server;
private String base;
private volatile int serverReceivedBytes;
private static ObservingOptions options() {
return
ObservingOptions.builder().metrics(EnumSet.of(ObservingOptions.MetricSet.CONTENT_LENGTH)).build();
}
private static MetricConfig config() {
return MetricConfig.builder().prefix("t").build();
}
private static Double total(final MeterRegistry meters, final String
name) {
final DistributionSummary s = meters.find(name).summary();
return s != null ? s.totalAmount() : null;
}
@BeforeEach
void setUp() throws IOException {
final ByteArrayOutputStream out = new ByteArrayOutputStream();
try (GZIPOutputStream gz = new GZIPOutputStream(out)) {
gz.write(BODY);
}
gzippedBody = out.toByteArray();
server = ServerBootstrap.bootstrap()
.setLocalAddress(InetAddress.getLoopbackAddress())
.setListenerPort(0)
.register("localhost", "/echo", (request, response, context)
-> {
serverReceivedBytes =
EntityUtils.toByteArray(request.getEntity()).length;
response.setCode(200);
})
.register("localhost", "/gzip", (request, response, context)
-> {
response.setCode(200);
response.setEntity(new ByteArrayEntity(gzippedBody,
ContentType.TEXT_PLAIN, "gzip"));
})
.create();
server.start();
base = "http://localhost:"; + server.getLocalPort();
}
@AfterEach
void tearDown() {
server.close(CloseMode.IMMEDIATE);
}
private MeterRegistry classic(final HttpPost post) throws IOException {
final MeterRegistry meters = new SimpleMeterRegistry();
final HttpClientBuilder builder = HttpClients.custom();
HttpClientObservationSupport.enable(builder, null, meters,
options(), config());
try (CloseableHttpClient client = builder.build()) {
client.execute(post, r -> {
EntityUtils.consume(r.getEntity());
return null;
});
}
return meters;
}
@Test
void requestCompressedWithEntityBuilderIsNotRecorded() throws Exception {
final HttpPost post = new HttpPost(base + "/echo");
post.setEntity(EntityBuilder.create().setBinary(BODY)
.setContentType(ContentType.APPLICATION_JSON).gzipCompressed().build());
final MeterRegistry meters = classic(post);
assertEquals(gzippedBody.length, serverReceivedBytes, "server
received the gzipped body");
assertNull(total(meters, REQUEST), "neither the decoded nor the wire
size is recorded");
}
@Test
void preEncodedRequestRecordsCompressedLength() throws Exception {
final HttpPost post = new HttpPost(base + "/echo");
post.setEntity(new ByteArrayEntity(gzippedBody,
ContentType.APPLICATION_JSON, "gzip"));
final MeterRegistry meters = classic(post);
assertEquals(gzippedBody.length, serverReceivedBytes);
assertEquals((double) gzippedBody.length, total(meters, REQUEST),
"compressed length, not 100000");
}
@Test
void transparentlyDecompressedResponseIsNotRecorded() throws Exception {
final MeterRegistry meters = classic(new HttpPost(base + "/gzip"));
assertNull(total(meters, RESPONSE), "Content-Length: 133 on the
wire, 100000 read by the app, nothing recorded");
}
@Test
void http2ResponseWithContentLengthIsNotRecorded() throws Exception {
final HttpAsyncServer h2 = H2ServerBootstrap.bootstrap()
.setVersionPolicy(HttpVersionPolicy.FORCE_HTTP_2)
.register("*", new
AsyncServerRequestHandler<Message<HttpRequest, byte[]>>() {
@Override
public AsyncRequestConsumer<Message<HttpRequest,
byte[]>> prepare(
final HttpRequest request, final EntityDetails
details, final HttpContext context) {
return new BasicRequestConsumer<>(details != null ?
new BasicAsyncEntityConsumer() : null);
}
@Override
public void handle(final Message<HttpRequest, byte[]>
message, final ResponseTrigger trigger,
final HttpContext context) throws
HttpException, IOException {
final BasicHttpResponse head = new
BasicHttpResponse(200);
head.addHeader("Content-Length",
Integer.toString(BODY.length));
trigger.submitResponse(new
BasicResponseProducer(head,
new BasicAsyncEntityProducer(BODY,
ContentType.APPLICATION_OCTET_STREAM)), context);
}
})
.create();
h2.start();
final ListenerEndpoint endpoint = h2.listen(
new InetSocketAddress(InetAddress.getLoopbackAddress(), 0),
URIScheme.HTTP).get();
final int port = ((InetSocketAddress)
endpoint.getAddress()).getPort();
final MeterRegistry meters = new SimpleMeterRegistry();
final HttpAsyncClientBuilder builder =
HttpAsyncClients.custom().setVersionPolicy(HttpVersionPolicy.FORCE_HTTP_2);
HttpClientObservationSupport.enable(builder, null, meters,
options(), config());
try (CloseableHttpAsyncClient client = builder.build()) {
client.start();
final SimpleHttpResponse response = client.execute(
SimpleRequestBuilder.get("http://localhost:"; + port +
"/").build(), null).get(20, TimeUnit.SECONDS);
assertEquals("100000",
response.getFirstHeader("Content-Length").getValue());
assertNull(total(meters, RESPONSE), "HTTP/2 response with
Content-Length: 100000, nothing recorded");
} finally {
h2.close(CloseMode.IMMEDIATE);
}
}
}
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
