AMBARI-21329 : No data on templated Grafana dashboards on HDF cluster. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8b90145c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8b90145c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8b90145c Branch: refs/heads/branch-feature-AMBARI-20859 Commit: 8b90145caadf73c550c2d209eec4044ba924de9a Parents: 8f80fe7 Author: Aravindan Vijayan <[email protected]> Authored: Fri Jun 23 14:27:08 2017 -0700 Committer: Aravindan Vijayan <[email protected]> Committed: Fri Jun 23 14:27:08 2017 -0700 ---------------------------------------------------------------------- .../timeline/AbstractTimelineMetricsSink.java | 25 ++++-- .../cache/HandleConnectExceptionTest.java | 86 +++++++++++++++++++- .../timeline/HBaseTimelineMetricStore.java | 6 +- .../timeline/TimelineMetricConfiguration.java | 10 +++ .../system/impl/AmbariMetricSinkImpl.java | 3 + .../system/impl/DatabaseMetricsSource.java | 4 +- .../metrics/system/impl/JvmMetricsSource.java | 12 ++- .../metrics/system/impl/MetricsServiceImpl.java | 5 +- 8 files changed, 133 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java index 7a84627..337f640 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java @@ -290,22 +290,29 @@ public abstract class AbstractTimelineMetricsSink { protected boolean emitMetrics(TimelineMetrics metrics) { String connectUrl; + boolean validCollectorHost = true; + if (isHostInMemoryAggregationEnabled()) { connectUrl = constructTimelineMetricUri("http", "localhost", String.valueOf(getHostInMemoryAggregationPort())); } else { String collectorHost = getCurrentCollectorHost(); + if (collectorHost == null) { + validCollectorHost = false; + } connectUrl = getCollectorUri(collectorHost); } - String jsonData = null; - LOG.debug("EmitMetrics connectUrl = " + connectUrl); - try { - jsonData = mapper.writeValueAsString(metrics); - } catch (IOException e) { - LOG.error("Unable to parse metrics", e); - } - if (jsonData != null) { - return emitMetricsJson(connectUrl, jsonData); + if (validCollectorHost) { + String jsonData = null; + LOG.debug("EmitMetrics connectUrl = " + connectUrl); + try { + jsonData = mapper.writeValueAsString(metrics); + } catch (IOException e) { + LOG.error("Unable to parse metrics", e); + } + if (jsonData != null) { + return emitMetricsJson(connectUrl, jsonData); + } } return false; } http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java index 4eb75eb..3be2162 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java @@ -45,7 +45,7 @@ import static org.powermock.api.easymock.PowerMock.replayAll; public class HandleConnectExceptionTest { private static final String COLLECTOR_URL = "collector"; private TestTimelineMetricsSink sink; - + @Before public void init(){ sink = new TestTimelineMetricsSink(); @@ -88,6 +88,17 @@ public class HandleConnectExceptionTest { } } + @Test + public void testEmitMetricsWithNullHost() { + TestTimelineMetricsSinkWithNullHost sinkWithNullHost = new TestTimelineMetricsSinkWithNullHost(); + + boolean success = sinkWithNullHost.emitMetrics(new TimelineMetrics()); + Assert.assertFalse(success); + + success = sinkWithNullHost.emitMetrics(new TimelineMetrics()); + Assert.assertTrue(success); + } + private class TestTimelineMetricsSink extends AbstractTimelineMetricsSink{ @Override protected String getCollectorUri(String host) { @@ -146,4 +157,77 @@ public class HandleConnectExceptionTest { } } + + private class TestTimelineMetricsSinkWithNullHost extends AbstractTimelineMetricsSink { + + int ctr = 0; + + @Override + protected String getCollectorUri(String host) { + return COLLECTOR_URL; + } + + @Override + protected String getCollectorProtocol() { + return "http"; + } + + @Override + protected String getCollectorPort() { + return "2181"; + } + + @Override + protected int getTimeoutSeconds() { + return 10; + } + + @Override + protected String getZookeeperQuorum() { + return "localhost:2181"; + } + + @Override + protected Collection<String> getConfiguredCollectorHosts() { + return Arrays.asList("localhost"); + } + + @Override + protected String getHostname() { + return "h1"; + } + + @Override + protected boolean isHostInMemoryAggregationEnabled() { + return false; + } + + @Override + protected int getHostInMemoryAggregationPort() { + return 0; + } + + @Override + public boolean emitMetrics(TimelineMetrics metrics) { + super.init(); + return super.emitMetrics(metrics); + } + + @Override + protected synchronized String findPreferredCollectHost() { + if (ctr == 0) { + ctr++; + return null; + } else { + return "localhost"; + } + } + + @Override + protected boolean emitMetricsJson(String connectUrl, String jsonData) { + return true; + } + + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index f984253..12c27a4 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -415,7 +415,11 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin throws SQLException, IOException { Map<String, Set<String>> hostedApps = metricMetadataManager.getHostedAppsCache(); - Map<String, Set<String>> instanceHosts = metricMetadataManager.getHostedInstanceCache(); + Map<String, Set<String>> instanceHosts = new HashMap<>(); + if (configuration.getTimelineMetricsMultipleClusterSupport()) { + instanceHosts = metricMetadataManager.getHostedInstanceCache(); + } + Map<String, Map<String, Set<String>>> instanceAppHosts = new HashMap<>(); if (MapUtils.isEmpty(instanceHosts)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java index 023465b..006a403 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java @@ -290,6 +290,9 @@ public class TimelineMetricConfiguration { public static final String TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES = "timeline.metrics.precision.table.hbase.hstore.blockingStoreFiles"; + public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS = + "timeline.metrics.support.multiple.clusters"; + public static final String HOST_APP_ID = "HOST"; public static final String DEFAULT_INSTANCE_PORT = "12001"; @@ -440,6 +443,13 @@ public class TimelineMetricConfiguration { return 3; } + public boolean getTimelineMetricsMultipleClusterSupport() { + if (metricsConf != null) { + return Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS, "false")); + } + return false; + } + public String getTimelineServiceRpcAddress() { String defaultRpcAddress = "0.0.0.0:60200"; if (metricsConf != null) { http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java index 7dec18f..5c5801d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java @@ -206,6 +206,7 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements timelineMetricsCache = new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval); if (CollectionUtils.isNotEmpty(collectorHosts)) { + LOG.info("Metric Sink initialized with collectorHosts : " + collectorHosts.toString()); isInitialized = true; } } @@ -234,6 +235,8 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements timelineMetrics.setMetrics(metricList); emitMetrics(timelineMetrics); } + } else { + LOG.debug("Metric Sink not yet initialized. Discarding metrics."); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java index 423b19f..a93b96c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java @@ -59,7 +59,7 @@ public class DatabaseMetricsSource extends AbstractMetricsSource { super.init(metricsConfig, sink); configuration = metricsConfig; initializeFilterSets(); - LOG.info("DatabaseMetricsSource initialized."); + LOG.info("Initialized Ambari DB Metrics Source..."); } /** @@ -83,11 +83,11 @@ public class DatabaseMetricsSource extends AbstractMetricsSource { @Override public void start() { - LOG.info("Starting Database Metrics source..."); ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("DatabaseMetricsSource-%d") .build(); executor = Executors.newSingleThreadExecutor(threadFactory); + LOG.info("Started Ambari DB Metrics source..."); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java index 348cc4f..ad05aa2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java @@ -60,20 +60,24 @@ public class JvmMetricsSource extends AbstractMetricsSource { registerAll(JVM_PREFIX + ".threads", new ThreadStatesGaugeSet(), registry); registry.register(JVM_PREFIX + ".file.open.descriptor.ratio", new FileDescriptorRatioGauge()); interval = Integer.parseInt(configuration.getProperty("interval", "10")); - LOG.info("JVM Metrics source initialized."); + LOG.info("Initialized JVM Metrics source..."); } @Override public void start() { - LOG.info("Starting JVM Metrics source..."); try { executor.scheduleWithFixedDelay(new Runnable() { @Override public void run() { - sink.publish(getMetrics()); - LOG.debug("********* Published JVM metrics to sink **********"); + try { + LOG.debug("Publishing JVM metrics to sink"); + sink.publish(getMetrics()); + } catch (Exception e) { + LOG.debug("Error in publishing JVM metrics to sink."); + } } }, interval, interval, TimeUnit.SECONDS); + LOG.info("Started JVM Metrics source..."); } catch (Exception e) { LOG.info("Throwing exception when starting metric source", e); } http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java index 0172b1d..ab8d779 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java @@ -61,9 +61,12 @@ public class MetricsServiceImpl implements MetricsService { Executors.newScheduledThreadPool(1).scheduleWithFixedDelay(new Runnable() { @Override public void run() { - LOG.info("Checking for metrics sink initialization"); if (!sink.isInitialized()) { + LOG.info("Attempting to initialize metrics sink"); initializeMetricsSink(); + if (sink.isInitialized()) { + LOG.info("Metric sink initialization successful"); + } } } }, 5, 5, TimeUnit.MINUTES);
