Repository: incubator-distributedlog Updated Branches: refs/heads/master 367335d5e -> 48d29356f
DL-133: Enable check style for distributedlog-benchmark module Author: Xi Liu <[email protected]> Reviewers: Sijie Guo <[email protected]> Closes #90 from xiliuant/xi/checkstyle_benchmark Project: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/commit/48d29356 Tree: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/tree/48d29356 Diff: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/diff/48d29356 Branch: refs/heads/master Commit: 48d29356f4406c66cb06653203625abf123f99bc Parents: 367335d Author: Xi Liu <[email protected]> Authored: Tue Dec 27 09:09:13 2016 -0800 Committer: Sijie Guo <[email protected]> Committed: Tue Dec 27 09:09:13 2016 -0800 ---------------------------------------------------------------------- distributedlog-benchmark/pom.xml | 34 +++++++++ .../distributedlog/benchmark/Benchmarker.java | 74 +++++++++++--------- .../benchmark/DLWriterWorker.java | 21 +++--- .../distributedlog/benchmark/ReaderWorker.java | 27 +++---- .../twitter/distributedlog/benchmark/Utils.java | 12 ++-- .../distributedlog/benchmark/Worker.java | 3 + .../distributedlog/benchmark/WriterWorker.java | 26 +++---- .../distributedlog/benchmark/package-info.java | 21 ++++++ .../stream/AbstractReaderBenchmark.java | 11 +-- .../benchmark/stream/AsyncReaderBenchmark.java | 11 ++- .../benchmark/stream/LedgerBatchReader.java | 7 +- .../benchmark/stream/LedgerReadBenchmark.java | 19 +++-- .../benchmark/stream/LedgerStreamReader.java | 11 ++- .../benchmark/stream/ReadMode.java | 3 + .../benchmark/stream/StreamBenchmark.java | 5 +- .../benchmark/stream/SyncReaderBenchmark.java | 11 +-- .../benchmark/stream/package-info.java | 21 ++++++ .../benchmark/utils/ShiftableRateLimiter.java | 3 +- .../benchmark/utils/package-info.java | 21 ++++++ 19 files changed, 229 insertions(+), 112 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/pom.xml ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/pom.xml b/distributedlog-benchmark/pom.xml index 49eeed9..bb4d3b9 100644 --- a/distributedlog-benchmark/pom.xml +++ b/distributedlog-benchmark/pom.xml @@ -112,6 +112,40 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <version>2.17</version> + <dependencies> + <dependency> + <groupId>com.puppycrawl.tools</groupId> + <artifactId>checkstyle</artifactId> + <version>6.19</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>distributedlog-build-tools</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <configuration> + <configLocation>distributedlog/checkstyle.xml</configLocation> + <suppressionsLocation>distributedlog/suppressions.xml</suppressionsLocation> + <consoleOutput>true</consoleOutput> + <failOnViolation>true</failOnViolation> + <includeResources>false</includeResources> + <includeTestSourceDirectory>true</includeTestSourceDirectory> + </configuration> + <executions> + <execution> + <phase>test-compile</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> </build> <profiles> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Benchmarker.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Benchmarker.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Benchmarker.java index ea5757d..87d3b53 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Benchmarker.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Benchmarker.java @@ -17,11 +17,20 @@ */ package com.twitter.distributedlog.benchmark; -import com.google.common.base.Preconditions; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + import com.twitter.distributedlog.DistributedLogConfiguration; import com.twitter.distributedlog.benchmark.utils.ShiftableRateLimiter; import com.twitter.finagle.stats.OstrichStatsReceiver; import com.twitter.finagle.stats.StatsReceiver; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.stats.NullStatsProvider; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.stats.StatsProvider; @@ -34,19 +43,14 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.TimeUnit; - +/** + * The launcher for benchmarks. + */ public class Benchmarker { - static final Logger logger = LoggerFactory.getLogger(Benchmarker.class); + private static final Logger logger = LoggerFactory.getLogger(Benchmarker.class); - final static String USAGE = "Benchmarker [-u <uri>] [-c <conf>] [-s serverset] [-m (read|write|dlwrite)]"; + static final String USAGE = "Benchmarker [-u <uri>] [-c <conf>] [-s serverset] [-m (read|write|dlwrite)]"; final String[] args; final Options options = new Options(); @@ -184,7 +188,7 @@ public class Benchmarker { } if (cmdline.hasOption("bs")) { batchSize = Integer.parseInt(cmdline.getOptionValue("bs")); - Preconditions.checkArgument("write" != mode, "batchSize supported only for mode=write"); + checkArgument("write" != mode, "batchSize supported only for mode=write"); } if (cmdline.hasOption("c")) { String configFile = cmdline.getOptionValue("c"); @@ -228,12 +232,12 @@ public class Benchmarker { batchFlushIntervalMicros = Integer.parseInt(cmdline.getOptionValue("bfi")); } - Preconditions.checkArgument(shardId >= 0, "shardId must be >= 0"); - Preconditions.checkArgument(numStreams > 0, "numStreams must be > 0"); - Preconditions.checkArgument(durationMins > 0, "durationMins must be > 0"); - Preconditions.checkArgument(streamPrefix != null, "streamPrefix must be defined"); - Preconditions.checkArgument(hostConnectionCoreSize > 0, "host connection core size must be > 0"); - Preconditions.checkArgument(hostConnectionLimit > 0, "host connection limit must be > 0"); + checkArgument(shardId >= 0, "shardId must be >= 0"); + checkArgument(numStreams > 0, "numStreams must be > 0"); + checkArgument(durationMins > 0, "durationMins must be > 0"); + checkArgument(streamPrefix != null, "streamPrefix must be defined"); + checkArgument(hostConnectionCoreSize > 0, "host connection core size must be > 0"); + checkArgument(hostConnectionLimit > 0, "host connection limit must be > 0"); if (cmdline.hasOption("p")) { statsProvider = ReflectionUtils.newInstance(cmdline.getOptionValue("p"), StatsProvider.class); @@ -275,14 +279,14 @@ public class Benchmarker { } Worker runWriter() { - Preconditions.checkArgument(!finagleNames.isEmpty() || !serversetPaths.isEmpty() || null != dlUri, + checkArgument(!finagleNames.isEmpty() || !serversetPaths.isEmpty() || null != dlUri, "either serverset paths, finagle-names or uri required"); - Preconditions.checkArgument(msgSize > 0, "messagesize must be greater than 0"); - Preconditions.checkArgument(rate > 0, "rate must be greater than 0"); - Preconditions.checkArgument(maxRate >= rate, "max rate must be greater than rate"); - Preconditions.checkArgument(changeRate >= 0, "change rate must be positive"); - Preconditions.checkArgument(changeRateSeconds >= 0, "change rate must be positive"); - Preconditions.checkArgument(concurrency > 0, "concurrency must be greater than 0"); + checkArgument(msgSize > 0, "messagesize must be greater than 0"); + checkArgument(rate > 0, "rate must be greater than 0"); + checkArgument(maxRate >= rate, "max rate must be greater than rate"); + checkArgument(changeRate >= 0, "change rate must be positive"); + checkArgument(changeRateSeconds >= 0, "change rate must be positive"); + checkArgument(concurrency > 0, "concurrency must be greater than 0"); ShiftableRateLimiter rateLimiter = new ShiftableRateLimiter(rate, maxRate, changeRate, changeRateSeconds, TimeUnit.SECONDS); @@ -357,12 +361,12 @@ public class Benchmarker { } Worker runDLWriter() throws IOException { - Preconditions.checkNotNull(dlUri, "dlUri must be defined"); - Preconditions.checkArgument(rate > 0, "rate must be greater than 0"); - Preconditions.checkArgument(maxRate >= rate, "max rate must be greater than rate"); - Preconditions.checkArgument(changeRate >= 0, "change rate must be positive"); - Preconditions.checkArgument(changeRateSeconds >= 0, "change rate must be positive"); - Preconditions.checkArgument(concurrency > 0, "concurrency must be greater than 0"); + checkNotNull(dlUri, "dlUri must be defined"); + checkArgument(rate > 0, "rate must be greater than 0"); + checkArgument(maxRate >= rate, "max rate must be greater than rate"); + checkArgument(changeRate >= 0, "change rate must be positive"); + checkArgument(changeRateSeconds >= 0, "change rate must be positive"); + checkArgument(concurrency > 0, "concurrency must be greater than 0"); ShiftableRateLimiter rateLimiter = new ShiftableRateLimiter(rate, maxRate, changeRate, changeRateSeconds, TimeUnit.SECONDS); @@ -379,10 +383,10 @@ public class Benchmarker { } Worker runReader() throws IOException { - Preconditions.checkArgument(!finagleNames.isEmpty() || !serversetPaths.isEmpty() || null != dlUri, + checkArgument(!finagleNames.isEmpty() || !serversetPaths.isEmpty() || null != dlUri, "either serverset paths, finagle-names or dlUri required"); - Preconditions.checkArgument(concurrency > 0, "concurrency must be greater than 0"); - Preconditions.checkArgument(truncationInterval > 0, "truncation interval should be greater than 0"); + checkArgument(concurrency > 0, "concurrency must be greater than 0"); + checkArgument(truncationInterval > 0, "truncation interval should be greater than 0"); return runReaderInternal(serversetPaths, finagleNames, truncationInterval); } @@ -393,7 +397,7 @@ public class Benchmarker { private Worker runReaderInternal(List<String> serversetPaths, List<String> finagleNames, int truncationInterval) throws IOException { - Preconditions.checkNotNull(dlUri); + checkNotNull(dlUri); int ssid = null == startStreamId ? shardId * numStreams : startStreamId; int esid = null == endStreamId ? (shardId + readersPerStream) * numStreams : endStreamId; http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/DLWriterWorker.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/DLWriterWorker.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/DLWriterWorker.java index 7a33cf4..152cd32 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/DLWriterWorker.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/DLWriterWorker.java @@ -17,7 +17,8 @@ */ package com.twitter.distributedlog.benchmark; -import com.google.common.base.Preconditions; +import static com.google.common.base.Preconditions.checkArgument; + import com.twitter.distributedlog.AsyncLogWriter; import com.twitter.distributedlog.DLSN; import com.twitter.distributedlog.DistributedLogConfiguration; @@ -29,12 +30,6 @@ import com.twitter.distributedlog.namespace.DistributedLogNamespaceBuilder; import com.twitter.distributedlog.util.FutureUtils; import com.twitter.distributedlog.util.SchedulerUtils; import com.twitter.util.FutureEventListener; -import org.apache.bookkeeper.stats.OpStatsLogger; -import org.apache.bookkeeper.stats.StatsLogger; -import org.apache.thrift.TException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.URI; import java.util.ArrayList; @@ -47,10 +42,18 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +/** + * The benchmark for core library writer. + */ public class DLWriterWorker implements Worker { - static final Logger LOG = LoggerFactory.getLogger(DLWriterWorker.class); + private static final Logger LOG = LoggerFactory.getLogger(DLWriterWorker.class); static final int BACKOFF_MS = 200; @@ -82,7 +85,7 @@ public class DLWriterWorker implements Worker { int writeConcurrency, int messageSizeBytes, StatsLogger statsLogger) throws IOException { - Preconditions.checkArgument(startStreamId <= endStreamId); + checkArgument(startStreamId <= endStreamId); this.streamPrefix = streamPrefix; this.startStreamId = startStreamId; this.endStreamId = endStreamId; http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/ReaderWorker.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/ReaderWorker.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/ReaderWorker.java index 62cd78f..91f36cd 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/ReaderWorker.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/ReaderWorker.java @@ -17,7 +17,8 @@ */ package com.twitter.distributedlog.benchmark; -import com.google.common.base.Preconditions; +import static com.google.common.base.Preconditions.checkArgument; + import com.google.common.base.Stopwatch; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.twitter.common.zookeeper.ServerSet; @@ -44,14 +45,6 @@ import com.twitter.util.Function; import com.twitter.util.Future; import com.twitter.util.FutureEventListener; import com.twitter.util.Promise; -import org.apache.bookkeeper.stats.Counter; -import org.apache.bookkeeper.stats.Gauge; -import org.apache.bookkeeper.stats.OpStatsLogger; -import org.apache.bookkeeper.stats.StatsLogger; -import org.apache.thrift.TException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.URI; import java.util.List; @@ -60,10 +53,20 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.stats.Counter; +import org.apache.bookkeeper.stats.Gauge; +import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +/** + * The benchmark for core library reader. + */ public class ReaderWorker implements Worker { - static final Logger LOG = LoggerFactory.getLogger(ReaderWorker.class); + private static final Logger LOG = LoggerFactory.getLogger(ReaderWorker.class); static final int BACKOFF_MS = 200; @@ -229,7 +232,7 @@ public class ReaderWorker implements Worker { boolean readFromHead, /* read from the earliest data of log */ StatsReceiver statsReceiver, StatsLogger statsLogger) throws IOException { - Preconditions.checkArgument(startStreamId <= endStreamId); + checkArgument(startStreamId <= endStreamId); this.streamPrefix = streamPrefix; this.startStreamId = startStreamId; this.endStreamId = endStreamId; @@ -281,7 +284,7 @@ public class ReaderWorker implements Worker { ServerSet local = this.serverSets[0].getServerSet(); ServerSet[] remotes = new ServerSet[this.serverSets.length - 1]; for (int i = 1; i < serverSets.length; i++) { - remotes[i-1] = serverSets[i].getServerSet(); + remotes[i - 1] = serverSets[i].getServerSet(); } builder = builder.serverSets(local, remotes); http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Utils.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Utils.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Utils.java index 8456a2d..f5c32db 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Utils.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Utils.java @@ -18,17 +18,19 @@ package com.twitter.distributedlog.benchmark; import com.twitter.distributedlog.benchmark.thrift.Message; +import java.nio.ByteBuffer; +import java.util.Random; import org.apache.thrift.TException; import org.apache.thrift.TSerializer; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.transport.TMemoryInputTransport; -import java.nio.ByteBuffer; -import java.util.Random; - +/** + * Utils for generating and parsing messages. + */ public class Utils { - static final Random random = new Random(System.currentTimeMillis()); + static final Random RAND = new Random(System.currentTimeMillis()); static final ThreadLocal<TSerializer> MSG_SERIALIZER = new ThreadLocal<TSerializer>() { @Override @@ -39,7 +41,7 @@ public class Utils { public static byte[] generateMessage(long requestMillis, int payLoadSize) throws TException { byte[] payload = new byte[payLoadSize]; - random.nextBytes(payload); + RAND.nextBytes(payload); Message msg = new Message(requestMillis, ByteBuffer.wrap(payload)); return MSG_SERIALIZER.get().serialize(msg); } http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Worker.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Worker.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Worker.java index 0492a03..6c60034 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Worker.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/Worker.java @@ -19,5 +19,8 @@ package com.twitter.distributedlog.benchmark; import java.io.Closeable; +/** + * Worker to run benchmark. + */ public interface Worker extends Closeable, Runnable { } http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/WriterWorker.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/WriterWorker.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/WriterWorker.java index a587375..46229b3 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/WriterWorker.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/WriterWorker.java @@ -17,7 +17,8 @@ */ package com.twitter.distributedlog.benchmark; -import com.google.common.base.Preconditions; +import static com.google.common.base.Preconditions.checkArgument; + import com.twitter.common.zookeeper.ServerSet; import com.twitter.distributedlog.DLSN; import com.twitter.distributedlog.benchmark.utils.ShiftableRateLimiter; @@ -30,17 +31,11 @@ import com.twitter.distributedlog.service.DistributedLogClientBuilder; import com.twitter.distributedlog.util.SchedulerUtils; import com.twitter.finagle.builder.ClientBuilder; import com.twitter.finagle.stats.StatsReceiver; -import com.twitter.finagle.thrift.ClientId$; import com.twitter.finagle.thrift.ClientId; +import com.twitter.finagle.thrift.ClientId$; import com.twitter.util.Duration$; import com.twitter.util.Future; import com.twitter.util.FutureEventListener; -import org.apache.bookkeeper.stats.OpStatsLogger; -import org.apache.bookkeeper.stats.StatsLogger; -import org.apache.thrift.TException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; @@ -49,9 +44,16 @@ import java.util.List; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +/** + * Benchmark for distributedlog proxy client. + */ public class WriterWorker implements Worker { static final Logger LOG = LoggerFactory.getLogger(WriterWorker.class); @@ -112,8 +114,8 @@ public class WriterWorker implements Worker { boolean enableBatching, int batchBufferSize, int batchFlushIntervalMicros) { - Preconditions.checkArgument(startStreamId <= endStreamId); - Preconditions.checkArgument(!finagleNames.isEmpty() || !serverSetPaths.isEmpty()); + checkArgument(startStreamId <= endStreamId); + checkArgument(!finagleNames.isEmpty() || !serverSetPaths.isEmpty()); this.streamPrefix = streamPrefix; this.dlUri = uri; this.startStreamId = startStreamId; @@ -207,7 +209,7 @@ public class WriterWorker implements Worker { ServerSet local = serverSets[0].getServerSet(); ServerSet[] remotes = new ServerSet[serverSets.length - 1]; for (int i = 1; i < serverSets.length; i++) { - remotes[i-1] = serverSets[i].getServerSet(); + remotes[i - 1] = serverSets[i].getServerSet(); } builder = builder.serverSets(local, remotes); } else { http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/package-info.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/package-info.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/package-info.java new file mode 100644 index 0000000..052a661 --- /dev/null +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Benchmarks for distributedlog. + */ +package com.twitter.distributedlog.benchmark; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/AbstractReaderBenchmark.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/AbstractReaderBenchmark.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/AbstractReaderBenchmark.java index 1fd9151..4d436ee 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/AbstractReaderBenchmark.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/AbstractReaderBenchmark.java @@ -24,7 +24,7 @@ import org.slf4j.LoggerFactory; abstract class AbstractReaderBenchmark extends StreamBenchmark { - protected static final Logger logger = LoggerFactory.getLogger(SyncReaderBenchmark.class); + private static final Logger logger = LoggerFactory.getLogger(SyncReaderBenchmark.class); protected ReadMode readMode = ReadMode.LATEST; protected long fromTxId = DistributedLogConstants.INVALID_TXID; @@ -32,9 +32,12 @@ abstract class AbstractReaderBenchmark extends StreamBenchmark { protected int batchSize = 1; protected AbstractReaderBenchmark() { - options.addOption("t", "tx-id", true, "Transaction ID to start read from when reading in mode 'position'"); - options.addOption("r", "rewind", true, "Time to rewind back to read from when reading in mode 'rewind' (in milliseconds)"); - options.addOption("m", "mode", true, "Read Mode : [oldest, latest, rewind, position]"); + options.addOption("t", "tx-id", true, + "Transaction ID to start read from when reading in mode 'position'"); + options.addOption("r", "rewind", true, + "Time to rewind back to read from when reading in mode 'rewind' (in milliseconds)"); + options.addOption("m", "mode", true, + "Read Mode : [oldest, latest, rewind, position]"); options.addOption("b", "batch-size", true, "Read batch size"); } http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/AsyncReaderBenchmark.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/AsyncReaderBenchmark.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/AsyncReaderBenchmark.java index 5c18705..86acdb6 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/AsyncReaderBenchmark.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/AsyncReaderBenchmark.java @@ -24,22 +24,21 @@ import com.twitter.distributedlog.DistributedLogManager; import com.twitter.distributedlog.LogRecordWithDLSN; import com.twitter.distributedlog.namespace.DistributedLogNamespace; import com.twitter.distributedlog.util.FutureUtils; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.OpStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.TimeUnit; - /** - * Benchmark on {@link com.twitter.distributedlog.AsyncLogReader} reading from a stream + * Benchmark on {@link com.twitter.distributedlog.AsyncLogReader} reading from a stream. */ public class AsyncReaderBenchmark extends AbstractReaderBenchmark { - static final Logger logger = LoggerFactory.getLogger(AsyncReaderBenchmark.class); + private static final Logger logger = LoggerFactory.getLogger(AsyncReaderBenchmark.class); @Override protected void benchmark(DistributedLogNamespace namespace, String logName, StatsLogger statsLogger) { http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerBatchReader.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerBatchReader.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerBatchReader.java index d58c9dc..6a11469 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerBatchReader.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerBatchReader.java @@ -17,6 +17,7 @@ */ package com.twitter.distributedlog.benchmark.stream; +import java.util.Enumeration; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; @@ -24,14 +25,12 @@ import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.ReadEntryListener import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Enumeration; - /** - * Read ledgers in batches + * Read ledgers in batches. */ public class LedgerBatchReader implements Runnable { - static final Logger logger = LoggerFactory.getLogger(LedgerBatchReader.class); + private static final Logger logger = LoggerFactory.getLogger(LedgerBatchReader.class); private final LedgerHandle lh; private final ReadEntryListener readEntryListener; http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerReadBenchmark.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerReadBenchmark.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerReadBenchmark.java index 0daffd5..d5ef5b2 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerReadBenchmark.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerReadBenchmark.java @@ -17,6 +17,8 @@ */ package com.twitter.distributedlog.benchmark.stream; +import static com.google.common.base.Charsets.UTF_8; + import com.google.common.base.Stopwatch; import com.twitter.distributedlog.BookKeeperClientBuilder; import com.twitter.distributedlog.DistributedLogManager; @@ -25,6 +27,9 @@ import com.twitter.distributedlog.ZooKeeperClient; import com.twitter.distributedlog.ZooKeeperClientBuilder; import com.twitter.distributedlog.metadata.BKDLConfig; import com.twitter.distributedlog.namespace.DistributedLogNamespace; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; @@ -34,18 +39,12 @@ import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static com.google.common.base.Charsets.UTF_8; - /** - * Benchmark ledger reading + * Benchmark ledger reading. */ public class LedgerReadBenchmark extends AbstractReaderBenchmark { - static final Logger logger = LoggerFactory.getLogger(AsyncReaderBenchmark.class); + private static final Logger logger = LoggerFactory.getLogger(AsyncReaderBenchmark.class); @Override protected void benchmark(DistributedLogNamespace namespace, String logName, StatsLogger statsLogger) { @@ -123,7 +122,7 @@ public class LedgerReadBenchmark extends AbstractReaderBenchmark { LedgerHandle lh = bk.openLedgerNoRecovery( lid, BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8)); logger.info("It took {} ms to open log segment {}", - new Object[] { stopwatch.elapsed(TimeUnit.MILLISECONDS), (lh.getLastAddConfirmed() + 1), segment }); + new Object[] { stopwatch.elapsed(TimeUnit.MILLISECONDS), (lh.getLastAddConfirmed() + 1), segment }); stopwatch.reset().start(); Runnable reader; if (streamRead) { @@ -143,7 +142,7 @@ public class LedgerReadBenchmark extends AbstractReaderBenchmark { } reader.run(); logger.info("It took {} ms to complete reading {} entries from log segment {}", - new Object[] { stopwatch.elapsed(TimeUnit.MILLISECONDS), (lh.getLastAddConfirmed() + 1), segment }); + new Object[] { stopwatch.elapsed(TimeUnit.MILLISECONDS), (lh.getLastAddConfirmed() + 1), segment }); } } catch (Exception e) { logger.error("Error on reading bk ", e); http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerStreamReader.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerStreamReader.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerStreamReader.java index 07af32d..e542af7 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerStreamReader.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/LedgerStreamReader.java @@ -17,6 +17,10 @@ */ package com.twitter.distributedlog.benchmark.stream; +import java.util.Enumeration; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.client.AsyncCallback; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.LedgerEntry; @@ -25,17 +29,12 @@ import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.ReadEntryListener import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Enumeration; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicLong; - /** * Reading ledger in a streaming way. */ public class LedgerStreamReader implements Runnable { - static final Logger logger = LoggerFactory.getLogger(LedgerStreamReader.class); + private static final Logger logger = LoggerFactory.getLogger(LedgerStreamReader.class); class PendingReadRequest implements AsyncCallback.ReadCallback { http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/ReadMode.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/ReadMode.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/ReadMode.java index a861647..280c9db 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/ReadMode.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/ReadMode.java @@ -17,6 +17,9 @@ */ package com.twitter.distributedlog.benchmark.stream; +/** + * The read mode for streaming read benchmark. + */ public enum ReadMode { OLDEST, LATEST, http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/StreamBenchmark.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/StreamBenchmark.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/StreamBenchmark.java index 89d64bb..1eff65a 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/StreamBenchmark.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/StreamBenchmark.java @@ -20,6 +20,8 @@ package com.twitter.distributedlog.benchmark.stream; import com.twitter.distributedlog.DistributedLogConfiguration; import com.twitter.distributedlog.namespace.DistributedLogNamespace; import com.twitter.distributedlog.namespace.DistributedLogNamespaceBuilder; +import java.io.File; +import java.net.URI; import org.apache.bookkeeper.stats.NullStatsProvider; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.stats.StatsProvider; @@ -31,9 +33,6 @@ import org.apache.commons.cli.Options; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.net.URI; - /** * Benchmark Streams. */ http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/SyncReaderBenchmark.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/SyncReaderBenchmark.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/SyncReaderBenchmark.java index 88755e2..3d5bd73 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/SyncReaderBenchmark.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/SyncReaderBenchmark.java @@ -22,18 +22,21 @@ import com.twitter.distributedlog.DistributedLogManager; import com.twitter.distributedlog.LogReader; import com.twitter.distributedlog.LogRecord; import com.twitter.distributedlog.namespace.DistributedLogNamespace; +import java.io.IOException; +import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.OpStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Benchmark on {@link com.twitter.distributedlog.LogReader} reading from a stream + * Benchmark on {@link com.twitter.distributedlog.LogReader} reading from a stream. */ public class SyncReaderBenchmark extends AbstractReaderBenchmark { + private static final Logger logger = LoggerFactory.getLogger(SyncReaderBenchmark.class); + public SyncReaderBenchmark() {} @Override http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/package-info.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/package-info.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/package-info.java new file mode 100644 index 0000000..d8e198c --- /dev/null +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/stream/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Stream level benchmarks. + */ +package com.twitter.distributedlog.benchmark.stream; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/utils/ShiftableRateLimiter.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/utils/ShiftableRateLimiter.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/utils/ShiftableRateLimiter.java index ba51e81..def0346 100644 --- a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/utils/ShiftableRateLimiter.java +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/utils/ShiftableRateLimiter.java @@ -18,13 +18,12 @@ package com.twitter.distributedlog.benchmark.utils; import com.google.common.util.concurrent.RateLimiter; - import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; /** - * A wrapper over rate limiter + * A wrapper over rate limiter. */ public class ShiftableRateLimiter implements Runnable { http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/48d29356/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/utils/package-info.java ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/utils/package-info.java b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/utils/package-info.java new file mode 100644 index 0000000..369b979 --- /dev/null +++ b/distributedlog-benchmark/src/main/java/com/twitter/distributedlog/benchmark/utils/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Utils for benchmarking. + */ +package com.twitter.distributedlog.benchmark.utils; \ No newline at end of file
