Repository: hive Updated Branches: refs/heads/llap 53b43cd44 -> ffed821ca
HIVE-13480 : Add hadoop2 metrics reporter for Codahale metrics (Sushanth Sowmyan, reviewed by Thejas Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0ef92b2c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0ef92b2c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0ef92b2c Branch: refs/heads/llap Commit: 0ef92b2c751f644c880e9e6b02cc81e1e6001504 Parents: fb230f9 Author: Sushanth Sowmyan <[email protected]> Authored: Fri Apr 22 13:57:05 2016 -0700 Committer: Sushanth Sowmyan <[email protected]> Committed: Fri Apr 22 13:57:05 2016 -0700 ---------------------------------------------------------------------- common/pom.xml | 5 +++++ .../common/metrics/metrics2/CodahaleMetrics.java | 16 ++++++++++++++++ .../common/metrics/metrics2/MetricsReporting.java | 3 ++- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 12 +++++++++++- pom.xml | 1 + 5 files changed, 35 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0ef92b2c/common/pom.xml ---------------------------------------------------------------------- diff --git a/common/pom.xml b/common/pom.xml index 1b9b4bc..67aab7c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -188,6 +188,11 @@ <artifactId>jackson-databind</artifactId> <version>${jackson.new.version}</version> </dependency> + <dependency> + <groupId>com.github.joshelser</groupId> + <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId> + <version>${dropwizard-metrics-hadoop-metrics2-reporter.version}</version> + </dependency> </dependencies> <profiles> http://git-wip-us.apache.org/repos/asf/hive/blob/0ef92b2c/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java index 7dea8de..4c43367 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java @@ -34,6 +34,7 @@ import com.codahale.metrics.jvm.GarbageCollectorMetricSet; import com.codahale.metrics.jvm.MemoryUsageGaugeSet; import com.codahale.metrics.jvm.ThreadStatesGaugeSet; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.joshelser.dropwizard.metrics.hadoop.HadoopMetrics2Reporter; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.cache.CacheBuilder; @@ -47,6 +48,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.common.metrics.common.MetricsScope; import org.apache.hadoop.hive.common.metrics.common.MetricsVariable; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -372,6 +374,20 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co jsonFileReporter.start(); reporters.add(jsonFileReporter); break; + case HADOOP2: + String applicationName = conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_COMPONENT_NAME.varname); + long reportingInterval = HiveConf.toTime( + conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_INTERVAL.varname), + TimeUnit.SECONDS, TimeUnit.SECONDS); + final HadoopMetrics2Reporter metrics2Reporter = HadoopMetrics2Reporter.forRegistry(metricRegistry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(DefaultMetricsSystem.initialize(applicationName), // The application-level name + applicationName, // Component name + applicationName, // Component description + "General"); // Name for each metric record + metrics2Reporter.start(reportingInterval, TimeUnit.SECONDS); + break; } } } http://git-wip-us.apache.org/repos/asf/hive/blob/0ef92b2c/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java index 643246f..e6d1f1b 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java @@ -23,5 +23,6 @@ package org.apache.hadoop.hive.common.metrics.metrics2; public enum MetricsReporting { JMX, CONSOLE, - JSON_FILE + JSON_FILE, + HADOOP2 } http://git-wip-us.apache.org/repos/asf/hive/blob/0ef92b2c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index c52b9d9..926806b 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1961,7 +1961,8 @@ public class HiveConf extends Configuration { "org.apache.hadoop.hive.common.metrics.LegacyMetrics"), "Hive metrics subsystem implementation class."), HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "JSON_FILE, JMX", - "Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics, comma separated list of JMX, CONSOLE, JSON_FILE"), + "Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics, " + + "comma separated list of JMX, CONSOLE, JSON_FILE, HADOOP2"), HIVE_METRICS_JSON_FILE_LOCATION("hive.service.metrics.file.location", "/tmp/report.json", "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, the location of local JSON metrics file. " + "This file will get overwritten at every interval."), @@ -1969,6 +1970,15 @@ public class HiveConf extends Configuration { new TimeValidator(TimeUnit.MILLISECONDS), "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, " + "the frequency of updating JSON metrics file."), + HIVE_METRICS_HADOOP2_INTERVAL("hive.service.metrics.hadoop2.frequency", "30s", + new TimeValidator(TimeUnit.SECONDS), + "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics HADOOP2 reporter, " + + "the frequency of updating the HADOOP2 metrics system."), + HIVE_METRICS_HADOOP2_COMPONENT_NAME("hive.service.metrics.hadoop2.component", + "hive", + "Component name to provide to Hadoop2 Metrics system. Ideally 'hivemetastore' for the MetaStore " + + " and and 'hiveserver2' for HiveServer2." + ), HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger", "The class responsible for logging client side performance metrics. \n" + "Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger"), http://git-wip-us.apache.org/repos/asf/hive/blob/0ef92b2c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 77cfaeb..5de35c8 100644 --- a/pom.xml +++ b/pom.xml @@ -128,6 +128,7 @@ <commons-dbcp.version>1.4</commons-dbcp.version> <derby.version>10.10.2.0</derby.version> <dropwizard.version>3.1.0</dropwizard.version> + <dropwizard-metrics-hadoop-metrics2-reporter.version>0.1.0</dropwizard-metrics-hadoop-metrics2-reporter.version> <guava.version>14.0.1</guava.version> <groovy.version>2.4.4</groovy.version> <hadoop.version>2.6.0</hadoop.version>
