Repository: incubator-gobblin Updated Branches: refs/heads/master 760da9a3e -> 940329e13
[GOBBLIN-358] Add logs for GobblinMetrics Closes #2231 from yukuai518/hang Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/940329e1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/940329e1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/940329e1 Branch: refs/heads/master Commit: 940329e132fa7622f3fe8f5892521f20e553c8fd Parents: 760da9a Author: Kuai Yu <[email protected]> Authored: Fri Jan 5 22:08:10 2018 -0800 Committer: Hung Tran <[email protected]> Committed: Fri Jan 5 22:08:10 2018 -0800 ---------------------------------------------------------------------- .../apache/gobblin/metrics/GobblinMetrics.java | 49 +++++++++++++------- 1 file changed, 31 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/940329e1/gobblin-metrics-libs/gobblin-metrics/src/main/java/org/apache/gobblin/metrics/GobblinMetrics.java ---------------------------------------------------------------------- diff --git a/gobblin-metrics-libs/gobblin-metrics/src/main/java/org/apache/gobblin/metrics/GobblinMetrics.java b/gobblin-metrics-libs/gobblin-metrics/src/main/java/org/apache/gobblin/metrics/GobblinMetrics.java index ae20031..703a603 100644 --- a/gobblin-metrics-libs/gobblin-metrics/src/main/java/org/apache/gobblin/metrics/GobblinMetrics.java +++ b/gobblin-metrics-libs/gobblin-metrics/src/main/java/org/apache/gobblin/metrics/GobblinMetrics.java @@ -26,6 +26,7 @@ import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.gobblin.metrics.reporter.FileFailureEventReporter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -393,30 +394,36 @@ public class GobblinMetrics { .getProperty(ConfigurationKeys.METRICS_REPORT_INTERVAL_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORT_INTERVAL)); ScheduledReporter.setReportingInterval(properties, reportInterval, reportTimeUnit); - // Build and start the JMX reporter - buildJmxMetricReporter(properties); - if (this.jmxReporter.isPresent()) { - LOGGER.info("Will start reporting metrics to JMX"); - this.jmxReporter.get().start(); - } + try { + // Build and start the JMX reporter + buildJmxMetricReporter(properties); + if (this.jmxReporter.isPresent()) { + LOGGER.info("Will start reporting metrics to JMX"); + this.jmxReporter.get().start(); + } - // Build all other reporters - buildFileMetricReporter(properties); - buildKafkaMetricReporter(properties); - buildGraphiteMetricReporter(properties); - buildInfluxDBMetricReporter(properties); - buildCustomMetricReporters(properties); - buildFileFailureEventReporter(properties); + // Build all other reporters + buildFileMetricReporter(properties); + buildKafkaMetricReporter(properties); + buildGraphiteMetricReporter(properties); + buildInfluxDBMetricReporter(properties); + buildCustomMetricReporters(properties); + buildFileFailureEventReporter(properties); - // Start reporters that implement org.apache.gobblin.metrics.report.ScheduledReporter - RootMetricContext.get().startReporting(); + // Start reporters that implement org.apache.gobblin.metrics.report.ScheduledReporter + RootMetricContext.get().startReporting(); - // Start reporters that implement com.codahale.metrics.ScheduledReporter - for (com.codahale.metrics.ScheduledReporter scheduledReporter : this.codahaleScheduledReporters) { - scheduledReporter.start(reportInterval, reportTimeUnit); + // Start reporters that implement com.codahale.metrics.ScheduledReporter + for (com.codahale.metrics.ScheduledReporter scheduledReporter : this.codahaleScheduledReporters) { + scheduledReporter.start(reportInterval, reportTimeUnit); + } + } catch (Exception e) { + LOGGER.error("Metrics reporting cannot be started due to {}", ExceptionUtils.getFullStackTrace(e)); + throw e; } this.metricsReportingStarted = true; + LOGGER.info("Metrics reporting has been started: GobblinMetrics {}", this.hashCode()); } /** @@ -428,6 +435,8 @@ public class GobblinMetrics { return; } + LOGGER.info("Metrics reporting will be stopped: GobblinMetrics {}", this.hashCode()); + // Stop the JMX reporter if (this.jmxReporter.isPresent()) { this.jmxReporter.get().stop(); @@ -446,9 +455,13 @@ public class GobblinMetrics { this.codahaleReportersCloser.close(); } catch (IOException ioe) { LOGGER.error("Failed to close metric output stream for job " + this.id, ioe); + } catch (Exception e) { + LOGGER.error("Failed to close metric output stream for job {} due to {}", this.id, ExceptionUtils.getFullStackTrace(e)); + throw e; } this.metricsReportingStarted = false; + LOGGER.info("Metrics reporting stopped successfully"); } private void buildFileMetricReporter(Properties properties) {
