This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-access.git
The following commit(s) were added to refs/heads/main by this push:
new dd320d7 Improve benchmark execution (#114)
dd320d7 is described below
commit dd320d782a1ba9e7d2e9e87e887f3e1a94523df2
Author: Christopher Tubbs <[email protected]>
AuthorDate: Fri Mar 13 20:20:11 2026 -0400
Improve benchmark execution (#114)
* Update README with more examples
* Allow controlling the specific benchmark pattern that applies to both
benchmark executions
* Do not fail the build when a benchmark pattern matches zero benchmarks
* Trigger profile with same property used to select benchmarks to run
for more intuitive execution (ensure same behavior with `-Dbenchmark`
as with `-Pbenchmark` when pattern is not specified)
* Print the benchmark options before execution, and include the JFR
output directory when JFR is enabled
---
README.md | 9 ++++--
modules/antlr4-example/pom.xml | 8 ++++++
.../antlr/AccessExpressionAntlrBenchmark.java | 27 +++++++++++++-----
modules/core/pom.xml | 17 ++++++++----
.../access/tests/AccessExpressionBenchmark.java | 32 +++++++++++++++-------
pom.xml | 8 +++++-
6 files changed, 74 insertions(+), 27 deletions(-)
diff --git a/README.md b/README.md
index 5052505..1aa81db 100644
--- a/README.md
+++ b/README.md
@@ -85,12 +85,15 @@ java
--module-path=modules/core/target/accumulo-access-core-$version.jar:modules
For an ANTLRv4 example, see its [README](modules/antlr4-example/README.md).
-## Running the Benchmark
+## Running the Benchmarks
-This project includes JMH Benchmarks. To run them, execute:
+This project includes JMH Benchmarks. To run them, execute the benchmark
profile:
```bash
-mvn clean verify -Pbenchmark
+mvn clean verify -Dbenchmark # run all benchmarks; alternatively, use
`-Pbenchmark`
+mvn clean verify -Dbenchmark=AccessExpressionAntlrBenchmark # run only the
Antlr benchmarks
+mvn clean verify -Dbenchmark='AccessExpressionBenchmark[.]measure.*Evaluation'
# run specific tests matching the pattern
+mvn clean verify -Dbenchmark -Dbenchmark.jfr # enable Java Flight Recorder for
AccessExpressionBenchmark; use -Dbenchmark.jfr.outputDir=/path/to/outputLocation
```
diff --git a/modules/antlr4-example/pom.xml b/modules/antlr4-example/pom.xml
index 10ee392..384c438 100644
--- a/modules/antlr4-example/pom.xml
+++ b/modules/antlr4-example/pom.xml
@@ -52,6 +52,11 @@
<artifactId>jmh-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<pluginManagement>
@@ -129,6 +134,9 @@
<classpath />
<argument>org.apache.accumulo.access.grammar.antlr.AccessExpressionAntlrBenchmark</argument>
</arguments>
+ <environmentVariables>
+ <ACCESS_BENCHMARK>${benchmark}</ACCESS_BENCHMARK>
+ </environmentVariables>
</configuration>
</execution>
</executions>
diff --git
a/modules/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/AccessExpressionAntlrBenchmark.java
b/modules/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/AccessExpressionAntlrBenchmark.java
index 77f65ff..e4d1e66 100644
---
a/modules/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/AccessExpressionAntlrBenchmark.java
+++
b/modules/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/AccessExpressionAntlrBenchmark.java
@@ -40,10 +40,12 @@ import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.NoBenchmarksException;
import org.openjdk.jmh.runner.Runner;
-import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
import org.openjdk.jmh.runner.options.TimeValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Benchmarks Access Expressions using JMH. To run, use the following commands.
@@ -57,6 +59,8 @@ import org.openjdk.jmh.runner.options.TimeValue;
*/
public class AccessExpressionAntlrBenchmark {
+ private static final Logger LOG =
LoggerFactory.getLogger(AccessExpressionAntlrBenchmark.class);
+
public static class EvaluatorTests {
AccessExpressionAntlrEvaluator evaluator;
@@ -170,15 +174,24 @@ public class AccessExpressionAntlrBenchmark {
state.loadData();
int numExpressions = state.getBytesExpressions().size();
+ LOG.info("Number of Expressions: {}", numExpressions);
- System.out.println("Number of Expressions: " + numExpressions);
+ var include = System.getenv().getOrDefault("ACCESS_BENCHMARK", "true");
+ if (include.equals("true")) {
+ include = "";
+ }
+ LOG.info("Benchmark include pattern: {}", include);
- Options opt = new
OptionsBuilder().include(AccessExpressionAntlrBenchmark.class.getSimpleName())
- .mode(Mode.Throughput).operationsPerInvocation(numExpressions)
-
.timeUnit(TimeUnit.MICROSECONDS).warmupTime(TimeValue.seconds(5)).warmupIterations(3)
- .measurementIterations(4).forks(3).build();
+ var opt = new OptionsBuilder().include(include).mode(Mode.Throughput)
+
.operationsPerInvocation(numExpressions).timeUnit(TimeUnit.MICROSECONDS)
+
.warmupTime(TimeValue.seconds(5)).warmupIterations(3).measurementIterations(4).forks(3)
+ .build();
- new Runner(opt).run();
+ try {
+ new Runner(opt).run();
+ } catch (NoBenchmarksException e) {
+ LOG.warn("No matching benchmarks");
+ }
}
}
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 9693339..e351503 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -28,10 +28,6 @@
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>accumulo-access-core</artifactId>
- <properties>
- <benchmark.jfr>false</benchmark.jfr>
- <benchmark.methods>.*</benchmark.methods>
- </properties>
<dependencies>
<dependency>
<groupId>com.github.spotbugs</groupId>
@@ -58,6 +54,11 @@
<artifactId>jmh-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
@@ -92,6 +93,9 @@
<name>benchmark</name>
</property>
</activation>
+ <properties>
+
<benchmark.jfr.outputDir>${project.build.directory}</benchmark.jfr.outputDir>
+ </properties>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
@@ -119,8 +123,9 @@
<argument>org.apache.accumulo.access.tests.AccessExpressionBenchmark</argument>
</arguments>
<environmentVariables>
- <ENABLE_JFR>${benchmark.jfr}</ENABLE_JFR>
- <ACCESS_BENCHMARK>${benchmark.methods}</ACCESS_BENCHMARK>
+ <ACCESS_BENCHMARK>${benchmark}</ACCESS_BENCHMARK>
+
<ACCESS_BENCHMARK_JFR>${benchmark.jfr}</ACCESS_BENCHMARK_JFR>
+
<ACCESS_BENCHMARK_JFR_DIR>${benchmark.jfr.outputDir}</ACCESS_BENCHMARK_JFR_DIR>
</environmentVariables>
</configuration>
</execution>
diff --git
a/modules/core/src/test/java/org/apache/accumulo/access/tests/AccessExpressionBenchmark.java
b/modules/core/src/test/java/org/apache/accumulo/access/tests/AccessExpressionBenchmark.java
index e153de9..586d640 100644
---
a/modules/core/src/test/java/org/apache/accumulo/access/tests/AccessExpressionBenchmark.java
+++
b/modules/core/src/test/java/org/apache/accumulo/access/tests/AccessExpressionBenchmark.java
@@ -40,11 +40,13 @@ import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.infra.Blackhole;
import org.openjdk.jmh.profile.JavaFlightRecorderProfiler;
+import org.openjdk.jmh.runner.NoBenchmarksException;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.RunnerException;
-import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
import org.openjdk.jmh.runner.options.OptionsBuilder;
import org.openjdk.jmh.runner.options.TimeValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@ -64,6 +66,8 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@SuppressFBWarnings(value = {"NP_UNWRITTEN_FIELD"}, justification = "Field is
written by Gson")
public class AccessExpressionBenchmark {
+ private static final Logger LOG =
LoggerFactory.getLogger(AccessExpressionBenchmark.class);
+
public static class VisibilityEvaluatorTests {
List<VisibilityEvaluator> evaluator;
List<byte[]> expressions;
@@ -253,23 +257,31 @@ public class AccessExpressionBenchmark {
state.loadData();
int numExpressions = state.getBytesExpressions().size();
+ LOG.info("Number of Expressions: {}", numExpressions);
- System.out.println("Number of Expressions: " + numExpressions);
-
- var include = System.getenv().getOrDefault("ACCESS_BENCHMARK",
- AccessExpressionBenchmark.class.getSimpleName());
+ var jfr =
Boolean.parseBoolean(System.getenv().getOrDefault("ACCESS_BENCHMARK_JFR",
"false"));
+ var jfrDir = System.getenv().getOrDefault("ACCESS_BENCHMARK_JFR_DIR",
+ System.getProperty("java.io.tmpdir"));
+ LOG.info("Java Flight Recorder: {}", jfr ? "enabled (outputDir=" + jfrDir
+ ")" : "disabled");
- var jfr = Boolean.parseBoolean(System.getenv().getOrDefault("ENABLE_JFR",
"false"));
+ var include = System.getenv().getOrDefault("ACCESS_BENCHMARK", "true");
+ if (include.equals("true")) {
+ include = "";
+ }
+ LOG.info("Benchmark include pattern: {}", include);
- ChainedOptionsBuilder builder = new
OptionsBuilder().include(include).mode(Mode.Throughput)
+ var builder = new OptionsBuilder().include(include).mode(Mode.Throughput)
.operationsPerInvocation(numExpressions).timeUnit(TimeUnit.MICROSECONDS)
.warmupTime(TimeValue.seconds(5)).warmupIterations(3).measurementIterations(4).forks(3);
if (jfr) {
- builder.addProfiler(JavaFlightRecorderProfiler.class,
- "dir=" + System.getProperty("java.io.tmpdir"));
+ builder.addProfiler(JavaFlightRecorderProfiler.class, "dir=" + jfrDir);
}
- new Runner(builder.build()).run();
+ try {
+ new Runner(builder.build()).run();
+ } catch (NoBenchmarksException e) {
+ LOG.warn("No matching benchmarks");
+ }
}
}
diff --git a/pom.xml b/pom.xml
index d4773d4..9fbb927 100644
--- a/pom.xml
+++ b/pom.xml
@@ -138,6 +138,7 @@ under the License.
<version.gson>2.13.2</version.gson>
<version.jmh>1.37</version.jmh>
<version.junit>6.0.3</version.junit>
+ <version.slf4j>2.0.17</version.slf4j>
</properties>
<dependencyManagement>
<dependencies>
@@ -181,10 +182,15 @@ under the License.
<artifactId>jmh-generator-annprocess</artifactId>
<version>${version.jmh}</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${version.slf4j}</version>
+ </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
- <version>2.0.17</version>
+ <version>${version.slf4j}</version>
</dependency>
</dependencies>
</dependencyManagement>
