yush1ga closed pull request #898: E2e in perf-consumer
URL: https://github.com/apache/incubator-pulsar/pull/898
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/pulsar-discovery-service/src/main/java/org/apache/pulsar/discovery/service/DiscoveryService.java
b/pulsar-discovery-service/src/main/java/org/apache/pulsar/discovery/service/DiscoveryService.java
index 76148dbad..0587345ae 100644
---
a/pulsar-discovery-service/src/main/java/org/apache/pulsar/discovery/service/DiscoveryService.java
+++
b/pulsar-discovery-service/src/main/java/org/apache/pulsar/discovery/service/DiscoveryService.java
@@ -104,13 +104,13 @@ public void startServer() throws Exception {
bootstrap.childHandler(new ServiceChannelInitializer(this, config,
false));
// Bind and start to accept incoming connections.
bootstrap.bind(config.getServicePort()).sync();
- LOG.info("Started Pulsar Discovery service on port {}",
config.getWebServicePort());
+ LOG.info("Started Pulsar Discovery service on port {}",
config.getServicePort());
if (config.isTlsEnabled()) {
ServerBootstrap tlsBootstrap = bootstrap.clone();
tlsBootstrap.childHandler(new ServiceChannelInitializer(this,
config, true));
tlsBootstrap.bind(config.getServicePortTls()).sync();
- LOG.info("Started Pulsar Discovery TLS service on port {}",
config.getWebServicePortTls());
+ LOG.info("Started Pulsar Discovery TLS service on port {}",
config.getServicePortTls());
}
}
diff --git
a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
index 8802609e0..ff69595b0 100644
---
a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
+++
b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
@@ -18,6 +18,8 @@
*/
package org.apache.pulsar.testclient;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
@@ -32,10 +34,11 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.LongAdder;
+import org.HdrHistogram.Histogram;
+import org.HdrHistogram.Recorder;
import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.ConsumerConfiguration;
-import org.apache.pulsar.client.api.ConsumerCryptoFailureAction;
import org.apache.pulsar.client.api.CryptoKeyReader;
import org.apache.pulsar.client.api.EncryptionKeyInfo;
import org.apache.pulsar.client.api.Message;
@@ -43,7 +46,6 @@
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.impl.PulsarClientImpl;
-import org.apache.pulsar.common.api.proto.PulsarApi.KeyValue;
import org.apache.pulsar.common.naming.DestinationName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,6 +63,10 @@
private static final LongAdder bytesReceived = new LongAdder();
private static final DecimalFormat dec = new DecimalFormat("0.000");
+ private static Recorder recorder = new
Recorder(TimeUnit.DAYS.toMillis(10), 5);
+ private static Recorder cumulativeRecorder = new
Recorder(TimeUnit.DAYS.toMillis(10), 5);
+
+
static class Arguments {
@Parameter(names = { "-h", "--help" }, description = "Help message",
help = true)
@@ -200,6 +206,10 @@ public void received(Consumer consumer, Message msg) {
limiter.acquire();
}
+ long latencyMillis = System.currentTimeMillis() -
msg.getPublishTime();
+ recorder.recordValue(latencyMillis);
+ cumulativeRecorder.recordValue(latencyMillis);
+
consumer.acknowledgeAsync(msg);
}
};
@@ -272,8 +282,18 @@ public EncryptionKeyInfo getPrivateKey(String keyName,
Map<String, String> keyMe
log.info("Start receiving from {} consumers on {} destinations",
arguments.numConsumers,
arguments.numDestinations);
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ public void run() {
+ printAggregatedStats();
+ }
+ });
+
+
long oldTime = System.nanoTime();
+ Histogram reportHistogram = null;
+
+
while (true) {
try {
Thread.sleep(10000);
@@ -287,11 +307,40 @@ public EncryptionKeyInfo getPrivateKey(String keyName,
Map<String, String> keyMe
double throughput = bytesReceived.sumThenReset() / elapsed * 8 /
1024 / 1024;
log.info("Throughput received: {} msg/s -- {} Mbit/s",
dec.format(rate), dec.format(throughput));
+
+ reportHistogram = recorder.getIntervalHistogram(reportHistogram);
+
+ log.info(
+ "Throughput received: {} msg/s -- {} Mbit/s --- Latency:
mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} -
Max: {}",
+ dec.format(rate), dec.format(throughput),
+ dec.format(reportHistogram.getMean()),
+ dec.format(reportHistogram.getValueAtPercentile(50)),
+ dec.format(reportHistogram.getValueAtPercentile(95)),
+ dec.format(reportHistogram.getValueAtPercentile(99)),
+ dec.format(reportHistogram.getValueAtPercentile(99.9)),
+ dec.format(reportHistogram.getValueAtPercentile(99.99)),
+ dec.format(reportHistogram.getMaxValue()));
+
+ reportHistogram.reset();
oldTime = now;
}
pulsarClient.close();
}
+ private static void printAggregatedStats() {
+ Histogram reportHistogram = cumulativeRecorder.getIntervalHistogram();
+
+ log.info(
+ "Aggregated latency stats --- Latency: mean: {} ms - med: {} -
95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - 99.999pct: {} - Max: {}",
+ dec.format(reportHistogram.getMean() / 1000.0),
+ dec.format(reportHistogram.getValueAtPercentile(50)),
+ dec.format(reportHistogram.getValueAtPercentile(95)),
+ dec.format(reportHistogram.getValueAtPercentile(99)),
+ dec.format(reportHistogram.getValueAtPercentile(99.9)),
+ dec.format(reportHistogram.getValueAtPercentile(99.99)),
+ dec.format(reportHistogram.getValueAtPercentile(99.999)),
+ dec.format(reportHistogram.getMaxValue()));
+ }
private static final Logger log =
LoggerFactory.getLogger(PerformanceConsumer.class);
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
