This is an automated email from the ASF dual-hosted git repository. twilmes pushed a commit to branch tp4-jmh in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 12f732872b7adf402a8484f88424d6cb6b8fb11c Author: Ted Wilmes <twil...@gmail.com> AuthorDate: Tue Apr 9 15:06:01 2019 -0500 Adding JMH based benchmarking module with initial traversal benchmarks based on Marko's processor benchmarks. --- java/machine/machine-perf-test/pom.xml | 142 +++++++++++++++++++++ .../benchmark/machine/PipesTraversalBenchmark.java | 27 ++++ .../machine/RxParallelTraversalBenchmark.java | 27 ++++ .../machine/RxSerialTraversalBenchmark.java | 27 ++++ .../benchmark/util/AbstractBenchmarkBase.java | 123 ++++++++++++++++++ .../benchmark/util/AbstractProcessorBenchmark.java | 69 ++++++++++ .../util/AbstractTraversalBenchmarkBase.java | 37 ++++++ .../tinkerpop/benchmark/util/ProcessorType.java | 23 ++++ .../tinkerpop/benchmark/util/WithProcessor.java | 32 +++++ java/machine/pom.xml | 1 + 10 files changed, 508 insertions(+) diff --git a/java/machine/machine-perf-test/pom.xml b/java/machine/machine-perf-test/pom.xml new file mode 100644 index 0000000..2bbfa98 --- /dev/null +++ b/java/machine/machine-perf-test/pom.xml @@ -0,0 +1,142 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>machine</artifactId> + <groupId>org.apache.tinkerpop</groupId> + <version>4.0.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <name>Apache TinkerPop :: Machine :: Machine-Perf-Test</name> + <artifactId>machine-perf-test</artifactId> + + <properties> + <jmh.version>1.21</jmh.version> + <!-- Skip benchmarks by default because they are time consuming. --> + <skipBenchmarks>true</skipBenchmarks> + <skipTests>${skipBenchmarks}</skipTests> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>machine-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>gremlin</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>pipes</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>rxjava</artifactId> + <version>${project.version}</version> + </dependency> + <!-- JMH --> + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-core</artifactId> + <version>${jmh.version}</version> + </dependency> + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-generator-annprocess</artifactId> + <version>${jmh.version}</version> + <scope>provided</scope> + </dependency> + <!-- TESTING --> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <version>${junit.version}</version> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>${junit.version}</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <artifactId>maven-deploy-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <testSourceDirectory>${project.build.sourceDirectory}</testSourceDirectory> + <testClassesDirectory>${project.build.outputDirectory}</testClassesDirectory> + <includes> + <include>**/*Benchmark*.java</include> + </includes> + <excludes> + <exclude>**/*$*.class</exclude> + <exclude>**/Abstract*</exclude> + <exclude>**/*_jmhType*</exclude> + </excludes> + <systemPropertyVariables> + <benchmarkReportDir>${project.build.directory}/reports/benchmark/</benchmarkReportDir> + <jvmArgs>-server -Xms2g -Xmx2g</jvmArgs> + <warmupIterations>10</warmupIterations> + <measureIterations>10</measureIterations> + <forks>2</forks> + </systemPropertyVariables> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <createDependencyReducedPom>false</createDependencyReducedPom> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <mainClass>org.openjdk.jmh.Main</mainClass> + </transformer> + </transformers> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file diff --git a/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/machine/PipesTraversalBenchmark.java b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/machine/PipesTraversalBenchmark.java new file mode 100644 index 0000000..8ce9ebc --- /dev/null +++ b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/machine/PipesTraversalBenchmark.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.benchmark.machine; + +import org.apache.tinkerpop.benchmark.util.ProcessorType; +import org.apache.tinkerpop.benchmark.util.AbstractTraversalBenchmarkBase; +import org.apache.tinkerpop.benchmark.util.WithProcessor; + +@WithProcessor(type = ProcessorType.PIPES) +public class PipesTraversalBenchmark extends AbstractTraversalBenchmarkBase { +} diff --git a/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/machine/RxParallelTraversalBenchmark.java b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/machine/RxParallelTraversalBenchmark.java new file mode 100644 index 0000000..8e8a34d --- /dev/null +++ b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/machine/RxParallelTraversalBenchmark.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.benchmark.machine; + +import org.apache.tinkerpop.benchmark.util.ProcessorType; +import org.apache.tinkerpop.benchmark.util.AbstractTraversalBenchmarkBase; +import org.apache.tinkerpop.benchmark.util.WithProcessor; + +@WithProcessor(type = ProcessorType.RX_PARALLEL) +public class RxParallelTraversalBenchmark extends AbstractTraversalBenchmarkBase { +} diff --git a/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/machine/RxSerialTraversalBenchmark.java b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/machine/RxSerialTraversalBenchmark.java new file mode 100644 index 0000000..d2d024d --- /dev/null +++ b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/machine/RxSerialTraversalBenchmark.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.benchmark.machine; + +import org.apache.tinkerpop.benchmark.util.ProcessorType; +import org.apache.tinkerpop.benchmark.util.AbstractTraversalBenchmarkBase; +import org.apache.tinkerpop.benchmark.util.WithProcessor; + +@WithProcessor(type = ProcessorType.RX_SERIAL) +public class RxSerialTraversalBenchmark extends AbstractTraversalBenchmarkBase { +} diff --git a/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/AbstractBenchmarkBase.java b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/AbstractBenchmarkBase.java new file mode 100644 index 0000000..3ff3a24 --- /dev/null +++ b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/AbstractBenchmarkBase.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.benchmark.util; + +import org.junit.jupiter.api.Test; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.io.File; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.concurrent.TimeUnit; + +/** + * Base class for all TinkerPop OpenJDK JMH benchmarks. Based upon Netty's approach to running JMH benchmarks + * from JUnit. + * + * @see <a href="http://netty.io/wiki/microbenchmarks.html"</a> + * + * @author Ted Wilmes (http://twilmes.org) + */ +@Warmup(iterations = AbstractBenchmarkBase.DEFAULT_WARMUP_ITERATIONS) +@Measurement(iterations = AbstractBenchmarkBase.DEFAULT_MEASURE_ITERATIONS) +@Fork(AbstractBenchmarkBase.DEFAULT_FORKS) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +public abstract class AbstractBenchmarkBase { + + protected static final int DEFAULT_WARMUP_ITERATIONS = 10; + protected static final int DEFAULT_MEASURE_ITERATIONS = 10; + protected static final int DEFAULT_FORKS = 2; + protected static final String DEFAULT_BENCHMARK_DIRECTORY = "./benchmarks/"; + protected static final String DEFAULT_JVM_ARGS = "-server -Xms2g -Xmx2g"; + + @Test + public void run() throws Exception { + final String className = getClass().getSimpleName(); + + final ChainedOptionsBuilder runnerOptions = new OptionsBuilder() + .include(".*" + className + ".*") + .mode(Mode.SampleTime) + .jvmArgs(getJvmArgs()); + + if (getWarmupIterations() > 0) { + runnerOptions.warmupIterations(getWarmupIterations()); + } + + if (getMeasureIterations() > 0) { + runnerOptions.measurementIterations(getMeasureIterations()); + } + + if (getForks() > 0) { + runnerOptions.forks(getForks()); + } + + if (getReportDir() != null) { + final String dtmStr = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); + final String filePath = getReportDir() + className + "-" + dtmStr + ".json"; + final File file = new File(filePath); + if (file.exists()) { + file.delete(); + } else { + file.getParentFile().mkdirs(); + file.createNewFile(); + } + + runnerOptions.resultFormat(ResultFormatType.JSON); + runnerOptions.result(filePath); + } + + new Runner(runnerOptions.build()).run(); + } + + protected int getWarmupIterations() { + return getIntProperty("warmupIterations", DEFAULT_WARMUP_ITERATIONS); + } + + protected int getMeasureIterations() { + return getIntProperty("measureIterations", DEFAULT_MEASURE_ITERATIONS); + } + + protected int getForks() { + return getIntProperty("forks", DEFAULT_FORKS); + } + + protected String getReportDir() { + return System.getProperty("benchmarkReportDir", DEFAULT_BENCHMARK_DIRECTORY); + } + + protected String[] getJvmArgs() { + return System.getProperty("jvmArgs", DEFAULT_JVM_ARGS).split(" "); + } + + private int getIntProperty(final String propertyName, final int defaultValue) { + final String propertyValue = System.getProperty(propertyName); + if(propertyValue == null) { + return defaultValue; + } + return Integer.valueOf(propertyValue); + } +} \ No newline at end of file diff --git a/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/AbstractProcessorBenchmark.java b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/AbstractProcessorBenchmark.java new file mode 100644 index 0000000..1da0d31 --- /dev/null +++ b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/AbstractProcessorBenchmark.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.benchmark.util; + +import org.apache.tinkerpop.language.gremlin.Gremlin; +import org.apache.tinkerpop.language.gremlin.TraversalSource; +import org.apache.tinkerpop.machine.Machine; +import org.apache.tinkerpop.machine.processor.pipes.PipesProcessor; +import org.apache.tinkerpop.machine.processor.rxjava.RxJavaProcessor; +import org.apache.tinkerpop.machine.species.LocalMachine; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@State(Scope.Thread) +public class AbstractProcessorBenchmark extends AbstractBenchmarkBase { + protected TraversalSource g; + protected List<Long> input; + + private final int ELEMENT_COUNT = 1000; + + @Setup + public void prepare() { + final WithProcessor[] withProcessors = this.getClass().getAnnotationsByType(WithProcessor.class); + WithProcessor withProcessor = withProcessors.length == 0 ? null : withProcessors[0]; + final ProcessorType processorType = withProcessor.type(); + + final Machine machine = LocalMachine.open(); + g = Gremlin.traversal(machine); + switch (processorType) { + case PIPES: + g = g.withProcessor(PipesProcessor.class); + case RX_SERIAL: + g = g.withProcessor(RxJavaProcessor.class); + break; + case RX_PARALLEL: + g = g.withProcessor(RxJavaProcessor.class, + Map.of(RxJavaProcessor.RXJAVA_THREADS, Runtime.getRuntime().availableProcessors() - 1)); + break; + default: + throw new RuntimeException("Unrecognized processor type"); + } + + input = new ArrayList<>(ELEMENT_COUNT); + for (long i = 0; i < ELEMENT_COUNT; i++) { + input.add(i+1); + } + } +} diff --git a/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/AbstractTraversalBenchmarkBase.java b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/AbstractTraversalBenchmarkBase.java new file mode 100644 index 0000000..7640509 --- /dev/null +++ b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/AbstractTraversalBenchmarkBase.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.benchmark.util; + +import org.apache.tinkerpop.language.gremlin.common.__; +import org.openjdk.jmh.annotations.Benchmark; + +import java.util.List; + +public class AbstractTraversalBenchmarkBase extends AbstractProcessorBenchmark { + + @Benchmark + public Object g_inject_unfold_incr_incr_incr_incr() { + return g.inject(input).unfold().incr().incr().incr().incr().toList(); + } + + @Benchmark + public List g_inject_unfold_repeat_times() { + return g.inject(input).unfold().repeat(__.incr()).times(4).toList(); + } +} diff --git a/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/ProcessorType.java b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/ProcessorType.java new file mode 100644 index 0000000..84bd628 --- /dev/null +++ b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/ProcessorType.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.benchmark.util; + +public enum ProcessorType { + PIPES, RX_SERIAL, RX_PARALLEL +} \ No newline at end of file diff --git a/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/WithProcessor.java b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/WithProcessor.java new file mode 100644 index 0000000..016da27 --- /dev/null +++ b/java/machine/machine-perf-test/src/main/java/org/apache/tinkerpop/benchmark/util/WithProcessor.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.benchmark.util; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface WithProcessor { + ProcessorType type() default ProcessorType.PIPES; +} diff --git a/java/machine/pom.xml b/java/machine/pom.xml index 59f1d06..40273c2 100644 --- a/java/machine/pom.xml +++ b/java/machine/pom.xml @@ -29,5 +29,6 @@ limitations under the License. <module>machine-test</module> <module>processor</module> <module>structure</module> + <module>machine-perf-test</module> </modules> </project> \ No newline at end of file