[geode-benchmarks] branch develop updated: GEODE-7554: Add retry mechanism for failed tests (#120)

[heybales]

This is an automated email from the ASF dual-hosted git repository.

heybales pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git


The following commit(s) were added to refs/heads/develop by this push:
     new bd2e000  GEODE-7554: Add retry mechanism for failed tests (#120)
bd2e000 is described below

commit bd2e0008335859ae012f383707a321299006fd02
Author: Helena Bales <hba...@pivotal.io>
AuthorDate: Wed Dec 11 14:13:41 2019 -0800

    GEODE-7554: Add retry mechanism for failed tests (#120)
    
    * write failed tests to a file
    * enable all benchmarks
    * update help message for --ci option in run_against_baseline.sh
---
 geode-benchmarks/build.gradle                                     | 5 -----
 .../main/java/org/apache/geode/perftest/analysis/Analyzer.java    | 8 ++++++++
 infrastructure/scripts/aws/run_against_baseline.sh                | 4 ++--
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/geode-benchmarks/build.gradle b/geode-benchmarks/build.gradle
index 0abe0dc..42cafe9 100644
--- a/geode-benchmarks/build.gradle
+++ b/geode-benchmarks/build.gradle
@@ -63,11 +63,6 @@ task benchmark(type: Test) {
   useJUnitPlatform()
   testLogging { exceptionFormat = 'full' }
 
-  exclude "**/*NonIndexedQueryBenchmark.class"
-  exclude "**/PartitionedFunctionExecutionBenchmark.class"
-  exclude "**/NoopBenchmark.class"
-  exclude "**/*LongBenchmark.class"
-
   forkEvery 1
 
   systemProperty 'TEST_HOSTS', project.findProperty('hosts')
diff --git 
a/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java 
b/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
index 13a90cc..9d69c75 100644
--- a/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
+++ b/harness/src/main/java/org/apache/geode/perftest/analysis/Analyzer.java
@@ -16,7 +16,9 @@ package org.apache.geode.perftest.analysis;
 
 import static java.lang.Double.isNaN;
 
+import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
 
@@ -70,6 +72,9 @@ public class Analyzer {
     benchmarkRunResult.writeResult(new PrintWriter(System.out));
     /* throw exc if failed? */
 
+    String errorFilePath = testResultArg + "/../../../failedTests";
+    BufferedWriter writer = new BufferedWriter(new FileWriter(errorFilePath, 
true));
+
     boolean isSignificantlyBetter = false;
     boolean isHighWaterCandidate = true;
     StringBuilder errorMessage = new StringBuilder();
@@ -79,12 +84,14 @@ public class Analyzer {
         if (isNaN(probeResult.baseline) || isNaN(probeResult.test)) {
           errorMessage.append("BENCHMARK FAILED: 
").append(benchmarkResult.name)
               .append(" missing result file.\n");
+          writer.append(benchmarkResult.name + "\n");
         } else if (probeResult.description.equals("average latency")) {
           if (probeResult.getDifference() > 0) {
             isHighWaterCandidate = false;
             if (probeResult.getDifference() >= 0.05) {
               errorMessage.append("BENCHMARK FAILED: 
").append(benchmarkResult.name)
                   .append(" average latency is 5% worse than baseline.\n");
+              writer.append(benchmarkResult.name + "\n");
             }
           } else if (probeResult.getDifference() <= -0.5) {
             isSignificantlyBetter = true;
@@ -92,6 +99,7 @@ public class Analyzer {
         }
       }
     }
+    writer.close();
 
     if (isCI && isHighWaterCandidate && isSignificantlyBetter) {
       System.out.println(
diff --git a/infrastructure/scripts/aws/run_against_baseline.sh 
b/infrastructure/scripts/aws/run_against_baseline.sh
index f4e54fa..87a202d 100755
--- a/infrastructure/scripts/aws/run_against_baseline.sh
+++ b/infrastructure/scripts/aws/run_against_baseline.sh
@@ -129,7 +129,7 @@ while (( "$#" )); do
       echo "-R|--baseline-geode-repo : Geode baseline repo (default: 
${DEFAULT_BASELINE_REPO})"
       echo "-B|--baseline-geode-branch : Geode baseline branch"
       echo "-m|--metadata : Test metadata to output to file, comma-delimited"
-      echo "--ci : Set if starting instances for Continuous Integration"
+      echo "--ci : Set if starting instances for Continuous Integration - used 
to retry failed tests and to name/target AWS instances"
       echo "-- : All subsequent arguments are passed to the benchmark task as 
arguments."
       echo "-h|--help|-? : This help message"
       exit 1
@@ -208,4 +208,4 @@ if [[ -z "${CI}" ]]; then
     ./analyze_tests.sh -o ${OUTPUT}
 else
     ./analyze_tests.sh -o ${OUTPUT} --ci
-fi
+fi
\ No newline at end of file