[kudu] branch master updated: RetryRule: don't report NTP desynchronization test failures

[adar]

This is an automated email from the ASF dual-hosted git repository.

adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new 2a0a5ef  RetryRule: don't report NTP desynchronization test failures
2a0a5ef is described below

commit 2a0a5ef2193a8e83bd8374772bc491594c1c0aa8
Author: Adar Dembo <a...@cloudera.com>
AuthorDate: Tue Apr 30 16:44:55 2019 -0700

    RetryRule: don't report NTP desynchronization test failures
    
    This mirrors the logic in build-support/report-test.sh which skips reporting
    of a failed test if there was an NTP clock synchronization issue.
    
    I went back and forth on whether to do this in RetryRule or ResultReporter,
    but settled on RetryRule as it seemed like the better place for "policy". I
    also figured spawning a subprocess is faster (and easier) than doing a
    primitive grep via Java APIs.
    
    I tested this by running a local test result server and injecting a failure
    into a test, once with the magic desynchronization string, and once without.
    I saw the failure get reported in the former and skipped in the latter.
    
    Change-Id: Ibba9768775f225425e98c3c36733aefcc7a42a11
    Reviewed-on: http://gerrit.cloudera.org:8080/13206
    Tested-by: Adar Dembo <a...@cloudera.com>
    Reviewed-by: Grant Henke <granthe...@apache.org>
---
 .../java/org/apache/kudu/test/junit/RetryRule.java | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git 
a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/junit/RetryRule.java 
b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/junit/RetryRule.java
index 80272fd..d420150 100644
--- 
a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/junit/RetryRule.java
+++ 
b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/junit/RetryRule.java
@@ -17,6 +17,7 @@
 package org.apache.kudu.test.junit;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 
 import org.apache.kudu.test.CapturingToFileLogAppender;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -162,6 +163,17 @@ public class RetryRule implements TestRule {
       reporter.tryReportResult(humanReadableTestName, result, logFile);
     }
 
+    private boolean wasClockUnsynchronized(File output) {
+      ProcessBuilder pb = new ProcessBuilder(ImmutableList.of(
+          "zgrep", "-q", "Clock considered unsynchronized", output.getPath()));
+      try {
+        Process p = pb.start();
+        return p.waitFor() == 0;
+      } catch (InterruptedException | IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
     private void doOneAttemptAndReport(int attempt) throws Throwable {
       try (CapturingToFileLogAppender capturer =
            new CapturingToFileLogAppender(/*useGzip=*/ true)) {
@@ -184,7 +196,15 @@ public class RetryRule implements TestRule {
             LOG.error("{}: failed attempt {}", humanReadableTestName, attempt, 
t);
           }
           capturer.finish();
-          report(ResultReporter.Result.FAILURE, capturer.getOutputFile());
+
+          // We sometimes have flaky infrastructure where NTP is broken. In 
that
+          // case do not report the test failure.
+          File output = capturer.getOutputFile();
+          if (wasClockUnsynchronized(output)) {
+            LOG.info("Not reporting test that failed due to NTP issues.");
+          } else {
+            report(ResultReporter.Result.FAILURE, output);
+          }
           throw t;
         }
       }