Author: tripod
Date: Tue Oct 15 23:49:47 2013
New Revision: 1532600
URL: http://svn.apache.org/r1532600
Log:
OAK-641: Improved benchmark tooling
- added very simple csv export
- include concurrency in test results
Modified:
jackrabbit/oak/trunk/oak-run/README.md
jackrabbit/oak/trunk/oak-run/run_concurrent.sh
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/Benchmark.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/ManyNodes.java
Modified: jackrabbit/oak/trunk/oak-run/README.md
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1532600&r1=1532599&r2=1532600&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/README.md (original)
+++ jackrabbit/oak/trunk/oak-run/README.md Tue Oct 15 23:49:47 2013
@@ -66,6 +66,7 @@ The following benchmark options (with de
--itemsToRead 1000 - Number of items to read
--bgReaders 20 - Number of background readers
--report false - Whether to output intermediate results
+ --csvFile <file> - Optional csv file to report the benchmark results
These options are passed to the test cases and repository fixtures
that need them. For example the Wikipedia dump option is needed by the
@@ -123,6 +124,7 @@ reported:
| Column | Description |
|-------------|-------------------------------------------------------|
+| C | concurrency level (only some tests) |
| min | minimum time (in ms) taken by a test run |
| 10% | time (in ms) in which the fastest 10% of test runs |
| 50% | time (in ms) taken by the median test run |
Modified: jackrabbit/oak/trunk/oak-run/run_concurrent.sh
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/run_concurrent.sh?rev=1532600&r1=1532599&r2=1532600&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/run_concurrent.sh (original)
+++ jackrabbit/oak/trunk/oak-run/run_concurrent.sh Tue Oct 15 23:49:47 2013
@@ -15,17 +15,32 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-AS_ADMIN=false
+USERS="false true"
RUNTIME=20
BENCH=ConcurrentReadAccessControlledTreeTest
-for fix in Oak-Tar Jackrabbit
+FIXS="Oak-Tar" # Jackrabbit"
+THREADS="1 2 4 8 10 12 14 16 18 20"
+
+
+LOG="bench_$(date +'%Y%m%d_%H%M%S').csv"
+echo "Benchmarks: $BENCH" > $LOG
+echo "Fixtures: $FIXS" >> $LOG
+echo "Users: $USERS" >> $LOG
+echo "Runtime: $RUNTIME" >> $LOG
+echo "Concurrency: $THREADS" >> $LOG
+echo "--------------------------------------" >> $LOG
+for user in $USERS
do
- for i in 1 2 4 8 10 12 14 16 18 20
- do
- echo "Executing benchmark with $i threads on $fix"
- rm -rf target/jackrabbit-*
- cmd="java -Xmx2048m -Druntime=$RUNTIME -jar
target/oak-run-0.10-SNAPSHOT.jar benchmark --bgReaders $i --runAsAdmin
$AS_ADMIN --report false $BENCH $fix"
- echo $cmd
- $cmd
+ echo "Executing benchmarks as admin: $user" | tee -a $LOG
+ echo "-----------------------------------------------------------" | tee
-a $LOG
+ for i in $THREADS
+ do
+ rm -rf target/Jackrabbit-* target/Oak-Tar-*
+ cmd="java -Xmx2048m -Druntime=$RUNTIME -jar
target/oak-run-*-SNAPSHOT.jar benchmark --csvFile $LOG --bgReaders $i
--runAsAdmin $user --report false $BENCH $FIXS"
+ echo $cmd
+ $cmd
done
done
+echo "-----------------------------------------"
+echo "Benchmark completed. see $LOG for details:"
+cat $LOG
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractTest.java?rev=1532600&r1=1532599&r2=1532600&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractTest.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractTest.java
Tue Oct 15 23:49:47 2013
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.benchmark;
+import java.io.PrintStream;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
@@ -66,6 +67,8 @@ abstract class AbstractTest extends Benc
private Profiler profiler;
+ private int concurrency;
+
protected static int getScale(int def) {
int scale = Integer.getInteger("scale", 0);
if (scale == 0) {
@@ -97,10 +100,15 @@ abstract class AbstractTest extends Benc
}
@Override
- public void run(Iterable<RepositoryFixture> fixtures) {
+ public void run(Iterable<RepositoryFixture> fixtures, PrintStream out) {
System.out.format(
- "# %-26.26s min 10%% 50%% 90%% max
N%n",
+ "# %-26.26s C min 10%% 50%% 90%% max
N%n",
toString());
+ if (out != null) {
+ out.format(
+ "# %-26.26s, C, min, 10%%, 50%%, 90%%,
max, N%n",
+ toString());
+ }
for (RepositoryFixture fixture : fixtures) {
try {
Repository[] cluster = fixture.setUpCluster(1);
@@ -109,14 +117,27 @@ abstract class AbstractTest extends Benc
DescriptiveStatistics statistics = runTest(cluster[0]);
if (statistics.getN() > 0) {
System.out.format(
- "%-28.28s %6.0f %6.0f %6.0f %6.0f %6.0f
%6d%n",
+ "%-28.28s %6d %6.0f %6.0f %6.0f %6.0f
%6.0f %6d%n",
fixture.toString(),
+ concurrency,
statistics.getMin(),
statistics.getPercentile(10.0),
statistics.getPercentile(50.0),
statistics.getPercentile(90.0),
statistics.getMax(),
statistics.getN());
+ if (out != null) {
+ out.format(
+ "%-28.28s, %6d, %6.0f, %6.0f, %6.0f,
%6.0f, %6.0f, %6d%n",
+ fixture.toString(),
+ concurrency,
+ statistics.getMin(),
+ statistics.getPercentile(10.0),
+ statistics.getPercentile(50.0),
+ statistics.getPercentile(90.0),
+ statistics.getMax(),
+ statistics.getN());
+ }
}
} finally {
fixture.tearDownCluster();
@@ -193,6 +214,7 @@ abstract class AbstractTest extends Benc
}
}
+ concurrency = this.threads.size();
this.threads = null;
this.sessions = null;
this.credentials = null;
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/Benchmark.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/Benchmark.java?rev=1532600&r1=1532599&r2=1532600&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/Benchmark.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/Benchmark.java
Tue Oct 15 23:49:47 2013
@@ -16,6 +16,8 @@
*/
package org.apache.jackrabbit.oak.benchmark;
+import java.io.PrintStream;
+
import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
public abstract class Benchmark {
@@ -26,7 +28,7 @@ public abstract class Benchmark {
*
* @param fixtures repository fixtures
*/
- public abstract void run(Iterable<RepositoryFixture> fixtures);
+ public abstract void run(Iterable<RepositoryFixture> fixtures, PrintStream
out);
public String toString() {
String name = getClass().getName();
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java?rev=1532600&r1=1532599&r2=1532600&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
Tue Oct 15 23:49:47 2013
@@ -17,6 +17,7 @@
package org.apache.jackrabbit.oak.benchmark;
import java.io.File;
+import java.io.PrintStream;
import java.util.List;
import java.util.Set;
@@ -25,6 +26,8 @@ import com.google.common.collect.Sets;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
+
+import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.benchmark.wikipedia.WikipediaImport;
import org.apache.jackrabbit.oak.fixture.JackrabbitRepositoryFixture;
import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
@@ -64,6 +67,8 @@ public class BenchmarkRunner {
OptionSpec<Boolean> report = parser.accepts("report", "Whether to
output intermediate results")
.withOptionalArg().ofType(Boolean.class)
.defaultsTo(Boolean.FALSE);
+ OptionSpec<File> csvFile = parser.accepts("csvFile", "File to write a
CSV version of the benchmark data.")
+ .withOptionalArg().ofType(File.class);
OptionSet options = parser.parse(args);
int cacheSize = cache.value(options);
@@ -161,8 +166,15 @@ public class BenchmarkRunner {
}
if (argset.isEmpty()) {
+ PrintStream out = null;
+ if (options.has(csvFile)) {
+ out = new
PrintStream(FileUtils.openOutputStream(csvFile.value(options), true));
+ }
for (Benchmark benchmark : benchmarks) {
- benchmark.run(fixtures);
+ benchmark.run(fixtures, out);
+ }
+ if (out != null) {
+ out.close();
}
} else {
System.err.println("Unknown arguments: " + argset);
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/ManyNodes.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/ManyNodes.java?rev=1532600&r1=1532599&r2=1532600&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/ManyNodes.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/ManyNodes.java
Tue Oct 15 23:49:47 2013
@@ -16,6 +16,8 @@
*/
package org.apache.jackrabbit.oak.benchmark;
+import java.io.PrintStream;
+
import javax.jcr.Node;
import javax.jcr.Repository;
import javax.jcr.RepositoryException;
@@ -27,7 +29,7 @@ import org.apache.jackrabbit.oak.fixture
public class ManyNodes extends Benchmark {
@Override
- public void run(Iterable<RepositoryFixture> fixtures) {
+ public void run(Iterable<RepositoryFixture> fixtures, PrintStream out) {
for (RepositoryFixture fixture : fixtures) {
System.out.println("ManyNodes test: " + fixture);
if (fixture.isAvailable(1)) {
