WireBaron closed pull request #7: GEODE-6086: Adding a tool to allow command
line analysis of test runs
URL: https://github.com/apache/geode-benchmarks/pull/7
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/harness/build.gradle b/harness/build.gradle
index 1658ecf..3dc5d44 100644
--- a/harness/build.gradle
+++ b/harness/build.gradle
@@ -28,6 +28,11 @@ repositories {
mavenCentral()
}
+task(analyzeRun, dependsOn: 'classes', type: JavaExec) {
+ main = 'org.apache.geode.perftest.analysis.Analyzer'
+ classpath = sourceSets.main.runtimeClasspath
+}
+
dependencies {
testCompile group: 'junit', name: 'junit', version: '4.12'
compile group: 'com.hierynomus', name: 'sshj', version: '0.26.0'
diff --git
a/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
b/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
new file mode 100644
index 0000000..f0d1988
--- /dev/null
+++ b/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional
information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
+ * or implied. See the License for the specific language governing permissions
and limitations under
+ * the License.
+ */
+package org.apache.geode.perftest.analysis;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import
org.apache.geode.perftest.yardstick.analysis.YardstickPercentileSensorParser;
+import
org.apache.geode.perftest.yardstick.analysis.YardstickThroughputSensorParser;
+
+public class Analyzer {
+
+ public static void main(String[] args) throws IOException {
+ if (args.length != 2) {
+ System.out.println("Analyzer takes two test output directories as
arguments, test results followed by baseline run result.");
+ System.exit(1);
+ return;
+ }
+
+ String testResultArg = args[0];
+ String baselineResultArg = args[1];
+
+ File testResultDir = new File(testResultArg);
+ File baselineResultDir = new File(baselineResultArg);
+
+ boolean valid = true;
+ if (!testResultDir.exists()) {
+ System.out.println("Unable to find test result directory: " +
testResultArg);
+ valid = false;
+ }
+ if (!baselineResultDir.exists()) {
+ System.out.println("Unable to find test result directory: " +
baselineResultArg);
+ valid = false;
+ }
+ if (!valid) {
+ System.exit(1);
+ return;
+ }
+
+ System.out.println("Running analyzer");
+ System.out.println("Comparing test result at " + testResultArg + " to
baseline at " + baselineResultArg);
+
+ BenchmarkRunAnalyzer analyzer = new BenchmarkRunAnalyzer();
+ analyzer.addProbe(new YardstickThroughputSensorParser());
+ analyzer.addProbe(new YardstickPercentileSensorParser());
+
+ analyzer.analyzeTestRun(testResultDir, baselineResultDir, new
PrintWriter(System.out));
+ }
+}
diff --git
a/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
b/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
index c84346b..19ffb58 100644
---
a/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
+++
b/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
@@ -15,13 +15,15 @@
package org.apache.geode.perftest.analysis;
import java.io.File;
-import java.io.FilenameFilter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.geode.perftest.yardstick.YardstickTask;
@@ -44,17 +46,8 @@
* </pre>
*/
public class BenchmarkRunAnalyzer {
- private final List<SensorData> benchmarks = new ArrayList<>();
private final List<ProbeResultParser> probes = new ArrayList<>();
- /**
- * Add a benchmark to be analyzed. The benchmark is expected to exist
- * in both result directories passed to {@link #analyzeTestRun(File, File,
Writer)}
- */
- public void addBenchmark(String name, String testResultDir, List<String>
nodeNames) {
- benchmarks.add(new SensorData(name, testResultDir, nodeNames));
- }
-
/**
* Add a probe to produce a comparison for. The probe expects to find output
files
* in the result directory for each node of each benchmark.
@@ -65,19 +58,26 @@ public void addProbe(ProbeResultParser probeResultParser) {
public void analyzeTestRun(File testResultDir, File baselineResultDir,
Writer output)
throws IOException {
+ List<File> benchmarkDirs = Arrays.asList(testResultDir.listFiles());
+
PrintWriter stream = new PrintWriter(output);
- for (SensorData benchmark : benchmarks) {
- stream.println("-- " + benchmark.benchmarkName + " --");
+ for (File testDir : benchmarkDirs) {
+ final List<File> yardstickOutputDirsForTest =
getYardstickOutputForBenchmarkDir(testDir);
+ if (yardstickOutputDirsForTest.isEmpty()) {
+ continue;
+ }
+ File baselineDir = new File(baselineResultDir, testDir.getName());
+ stream.println("-- " + testDir.getName() + " --");
for (ProbeResultParser probe : probes) {
stream.println(probe.getResultDescription());
- for (String node : benchmark.nodesToParse) {
- probe.parseResults(getBenchmarkOutputDir(testResultDir, benchmark,
node));
+ for (File outputDirectory : yardstickOutputDirsForTest) {
+ probe.parseResults(outputDirectory);
}
double testResult = probe.getProbeResult();
stream.println("Result: " + String.valueOf(testResult));
probe.reset();
- for (String node : benchmark.nodesToParse) {
- probe.parseResults(getBenchmarkOutputDir(baselineResultDir,
benchmark, node));
+ for (File outputDirectory :
getYardstickOutputForBenchmarkDir(baselineDir)) {
+ probe.parseResults(outputDirectory);
}
double baselineResult = probe.getProbeResult();
stream.println("Baseline: " + String.valueOf(baselineResult));
@@ -89,35 +89,10 @@ public void analyzeTestRun(File testResultDir, File
baselineResultDir, Writer ou
stream.flush();
}
- private File getBenchmarkOutputDir(File testResultDir, SensorData benchmark,
- String node) {
- File benchmarkDir = new File(testResultDir,
benchmark.benchmarkSubdirectory);
- File nodeDir = new File(benchmarkDir, node);
-
- File[] files = nodeDir.listFiles((dir, name) ->
name.contains(YardstickTask.YARDSTICK_OUTPUT));
-
- if(files == null || files.length != 1) {
- throw new IllegalStateException("Expected at least one subdirectory in "
+ nodeDir
- + " with the name *" +YardstickTask.YARDSTICK_OUTPUT);
- }
-
- return files[0];
+ private List<File> getYardstickOutputForBenchmarkDir(File benchmarkDir)
throws IOException {
+ return Files.walk(benchmarkDir.toPath(), 2)
+ .filter(path ->
path.toString().endsWith(YardstickTask.YARDSTICK_OUTPUT))
+ .map(Path::toFile)
+ .collect(Collectors.toList());
}
-
- // TODO: depending on how run output is stored, this data may be excessive
or insufficient
- // The present assumption is each benchmark contains an arbitrarily named
result directory
- // containing subdirectories for each node. Those subdirectories then
contain the probe output
- // files for the run, for that node.
- private static class SensorData {
- private final String benchmarkName;
- private final String benchmarkSubdirectory;
- private final List<String> nodesToParse;
-
- public SensorData(String benchmarkName, String benchmarkSubdirectory,
List<String> nodesNames) {
- this.benchmarkName = benchmarkName;
- this.benchmarkSubdirectory = benchmarkSubdirectory;
- this.nodesToParse = nodesNames;
- }
- }
-
}
diff --git
a/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
b/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
index 387c198..e838733 100644
---
a/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
+++
b/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
@@ -23,8 +23,6 @@
import java.io.PrintStream;
import java.io.StringReader;
import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Scanner;
import org.junit.Assert;
@@ -32,7 +30,6 @@
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import org.apache.geode.perftest.analysis.BenchmarkRunAnalyzer;
import
org.apache.geode.perftest.yardstick.analysis.YardstickPercentileSensorParser;
import
org.apache.geode.perftest.yardstick.analysis.YardstickThroughputSensorParser;
@@ -47,6 +44,8 @@ public void verifyResultHarvester() throws IOException {
final File testBenchmarkA2 =
temporaryFolder.newFolder("testFolder","BenchmarkA","client2",
"20181121-111516-yardstick-output");
final File testBenchmarkB1 =
temporaryFolder.newFolder("testFolder","BenchmarkB","client1",
"20181121-111516-yardstick-output");
final File testBenchmarkB2 =
temporaryFolder.newFolder("testFolder","BenchmarkB","client2",
"20181121-111516-yardstick-output");
+ temporaryFolder.newFolder(testFolder.getName(), "junkfolder");
+ new File(testFolder, "junkfile").createNewFile();
final File baseFolder = temporaryFolder.newFolder("baseFolder");
final File baseBenchmarkA1 =
temporaryFolder.newFolder("baseFolder","BenchmarkA","client1",
"20181121-111516-yardstick-output");
final File baseBenchmarkA2 =
temporaryFolder.newFolder("baseFolder","BenchmarkA","client2",
"20181121-111516-yardstick-output");
@@ -71,13 +70,7 @@ public void verifyResultHarvester() throws IOException {
populateThroughputCSV(baseBenchmarkB2, new double[] {10, 15, 20, 25, 30});
// Avg 20
populatePercentileCSV(baseBenchmarkB2, new double[] {0, 0, 0, 99, 1});
// 300
- List<String> nodes = new ArrayList(2);
- nodes.add("client1");
- nodes.add("client2");
-
BenchmarkRunAnalyzer harvester = new BenchmarkRunAnalyzer();
- harvester.addBenchmark("Alpha", "BenchmarkA", nodes);
- harvester.addBenchmark("Beta", "BenchmarkB", nodes);
harvester.addProbe(new YardstickThroughputSensorParser());
harvester.addProbe(new YardstickPercentileSensorParser());
@@ -88,8 +81,8 @@ public void verifyResultHarvester() throws IOException {
System.out.println(writer.toString());
BufferedReader resultReader = new BufferedReader(new
StringReader(writer.toString()));
- validatedBenchmark(resultReader, "Alpha", 20, 300, 25, 200);
- validatedBenchmark(resultReader, "Beta", 25, 400, 20, 400);
+ validatedBenchmark(resultReader, "BenchmarkA", 20, 300, 25, 200);
+ validatedBenchmark(resultReader, "BenchmarkB", 25, 400, 20, 400);
}
private void populateThroughputCSV(File targetDirectory, double[]
perSecondThroughputs)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
