Repository: ambari Updated Branches: refs/heads/trunk 2ff332504 -> 51480ecac
AMBARI-16123 Reduce error logs on the Sink path if AMS is down (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/51480eca Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/51480eca Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/51480eca Branch: refs/heads/trunk Commit: 51480ecaccae514035d3fae2b0a4757f51152567 Parents: 2ff3325 Author: Dmytro Sen <[email protected]> Authored: Thu Apr 28 20:33:22 2016 +0300 Committer: Dmytro Sen <[email protected]> Committed: Thu Apr 28 20:56:48 2016 +0300 ---------------------------------------------------------------------- .../timeline/AbstractTimelineMetricsSink.java | 25 ++++++++++++++++---- .../cache/HandleConnectExceptionTest.java | 23 +++++++++++++----- 2 files changed, 37 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/51480eca/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java index 5a532c5..217f265 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java @@ -38,6 +38,7 @@ import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; import java.security.KeyStore; +import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractTimelineMetricsSink { public static final String TAGS_FOR_PREFIX_PROPERTY_PREFIX = "tagsForPrefix."; @@ -57,6 +58,9 @@ public abstract class AbstractTimelineMetricsSink { public static final String SSL_KEYSTORE_TYPE_PROPERTY = "truststore.type"; public static final String SSL_KEYSTORE_PASSWORD_PROPERTY = "truststore.password"; + protected static final AtomicInteger failedCollectorConnectionsCounter = new AtomicInteger(0); + public static int NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS = 100; + private SSLSocketFactory sslSocketFactory; protected final Log LOG; @@ -109,9 +113,12 @@ public abstract class AbstractTimelineMetricsSink { } } cleanupInputStream(connection.getInputStream()); + //reset failedCollectorConnectionsCounter to "0" + failedCollectorConnectionsCounter.set(0); } catch (IOException ioe) { StringBuilder errorMessage = - new StringBuilder("Unable to connect to collector, " + connectUrl + "\n"); + new StringBuilder("Unable to connect to collector, " + connectUrl + "\n" + + "This exceptions will be ignored for next " + NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS + " times\n"); try { if ((connection != null)) { errorMessage.append(cleanupInputStream(connection.getErrorStream())); @@ -119,12 +126,20 @@ public abstract class AbstractTimelineMetricsSink { } catch (IOException e) { //NOP } - if (LOG.isDebugEnabled()) { - LOG.debug(errorMessage, ioe); + + if (failedCollectorConnectionsCounter.getAndIncrement() == 0) { + if (LOG.isDebugEnabled()) { + LOG.debug(errorMessage, ioe); + } else { + LOG.info(errorMessage); + } + throw new UnableToConnectException(ioe).setConnectUrl(connectUrl); } else { - LOG.info(errorMessage); + failedCollectorConnectionsCounter.compareAndSet(NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS, 0); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Ignoring %s AMS connection exceptions", NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS)); + } } - throw new UnableToConnectException(ioe).setConnectUrl(connectUrl); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/51480eca/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java index 4c1a2cb..ad7b1ac 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java @@ -23,6 +23,7 @@ import java.net.HttpURLConnection; import java.net.URL; import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; import org.apache.hadoop.metrics2.sink.timeline.UnableToConnectException; import org.junit.Assert; @@ -50,12 +51,12 @@ public class HandleConnectExceptionTest { OutputStream os = createNiceMock(OutputStream.class); HttpURLConnection connection = createNiceMock(HttpURLConnection.class); URL url = createNiceMock(URL.class); - + AbstractTimelineMetricsSink.NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS = 2; try { - expectNew(URL.class, "collector").andReturn(url); - expect(url.openConnection()).andReturn(connection).once(); - expect(connection.getOutputStream()).andReturn(os).once(); - expect(connection.getResponseCode()).andThrow(new IOException()); + expectNew(URL.class, "collector").andReturn(url).anyTimes(); + expect(url.openConnection()).andReturn(connection).anyTimes(); + expect(connection.getOutputStream()).andReturn(os).anyTimes(); + expect(connection.getResponseCode()).andThrow(new IOException()).anyTimes(); replayAll(); } catch (Exception e) { @@ -65,8 +66,18 @@ public class HandleConnectExceptionTest { @Test public void handleTest(){ - try{ + emitMetricsWithExpectedException(new TimelineMetrics()); + try { sink.emitMetrics(new TimelineMetrics()); + } catch (Exception e) { + Assert.fail("There should be no exception"); + } + emitMetricsWithExpectedException(new TimelineMetrics()); + } + + private void emitMetricsWithExpectedException(TimelineMetrics timelineMetrics) { + try{ + sink.emitMetrics(timelineMetrics); Assert.fail(); }catch(UnableToConnectException e){ Assert.assertEquals(COLLECTOR_URL, e.getConnectUrl());
