Repository: oozie Updated Branches: refs/heads/master bf8329c2f -> 67563df53
OOZIE-2251 Expose instrumental matrices in Realtime Graphing tool (nperiwal via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/67563df5 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/67563df5 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/67563df5 Branch: refs/heads/master Commit: 67563df53f2704476c81e85acc985b3522999e2a Parents: bf8329c Author: Robert Kanter <[email protected]> Authored: Mon Oct 19 17:02:02 2015 -0700 Committer: Robert Kanter <[email protected]> Committed: Mon Oct 19 17:02:02 2015 -0700 ---------------------------------------------------------------------- core/pom.xml | 38 +++++++ .../service/MetricsInstrumentationService.java | 1 + .../org/apache/oozie/util/Instrumentation.java | 4 + .../oozie/util/MetricsInstrumentation.java | 101 +++++++++++++++++++ core/src/main/resources/oozie-default.xml | 33 ++++++ .../oozie/util/TestMetricsInstrumentation.java | 8 ++ docs/src/site/twiki/AG_Install.twiki | 37 +++++++ pom.xml | 26 +++++ release-log.txt | 1 + webapp/pom.xml | 5 + 10 files changed, 254 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/67563df5/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index f5f89af..b063dab 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -36,6 +36,44 @@ <dependencies> <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-core</artifactId> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-graphite</artifactId> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-ganglia</artifactId> + </dependency> + + <dependency> + <groupId>info.ganglia.gmetric4j</groupId> + <artifactId>gmetric4j</artifactId> + <exclusions> + <exclusion> + <groupId>org.acplt</groupId> + <artifactId>oncrpc</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/oozie/blob/67563df5/core/src/main/java/org/apache/oozie/service/MetricsInstrumentationService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/MetricsInstrumentationService.java b/core/src/main/java/org/apache/oozie/service/MetricsInstrumentationService.java index 2a00a73..b57a2a5 100644 --- a/core/src/main/java/org/apache/oozie/service/MetricsInstrumentationService.java +++ b/core/src/main/java/org/apache/oozie/service/MetricsInstrumentationService.java @@ -52,6 +52,7 @@ public class MetricsInstrumentationService extends InstrumentationService { @Override public void destroy() { isEnabled = false; + instrumentation.stop(); instrumentation = null; } http://git-wip-us.apache.org/repos/asf/oozie/blob/67563df5/core/src/main/java/org/apache/oozie/util/Instrumentation.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/Instrumentation.java b/core/src/main/java/org/apache/oozie/util/Instrumentation.java index 4eb6386..fa1e92a 100644 --- a/core/src/main/java/org/apache/oozie/util/Instrumentation.java +++ b/core/src/main/java/org/apache/oozie/util/Instrumentation.java @@ -801,4 +801,8 @@ public class Instrumentation { return samplers; } + public void stop() { + + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/67563df5/core/src/main/java/org/apache/oozie/util/MetricsInstrumentation.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/MetricsInstrumentation.java b/core/src/main/java/org/apache/oozie/util/MetricsInstrumentation.java index 185b67e..70c894f 100644 --- a/core/src/main/java/org/apache/oozie/util/MetricsInstrumentation.java +++ b/core/src/main/java/org/apache/oozie/util/MetricsInstrumentation.java @@ -23,7 +23,11 @@ import com.codahale.metrics.Counter; import com.codahale.metrics.ExponentiallyDecayingReservoir; import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.ganglia.GangliaReporter; +import com.codahale.metrics.graphite.Graphite; +import com.codahale.metrics.graphite.GraphiteReporter; import com.codahale.metrics.json.MetricsModule; import com.codahale.metrics.jvm.MemoryUsageGaugeSet; import com.fasterxml.jackson.core.JsonProcessingException; @@ -32,9 +36,14 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import info.ganglia.gmetric4j.gmetric.GMetric; +import org.apache.oozie.service.ConfigurationService; import java.io.IOException; import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.URL; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -65,15 +74,84 @@ public class MetricsInstrumentation extends Instrumentation { private Lock countersLock; private Lock histogramsLock; + public static final String EXTERNAL_MONITORING_ENABLE = "oozie.external_monitoring.enable"; + public static final String EXTERNAL_MONITORING_TYPE = "oozie.external_monitoring.type"; + public static final String EXTERNAL_MONITORING_ADDRESS = "oozie.external_monitoring.address"; + public static final String EXTERNAL_MONITORING_PREFIX = "oozie.external_monitoring.metricPrefix"; + public static final String EXTERNAL_MONITORING_INTERVAL = "oozie.external_monitoring.reporterIntervalSecs"; + public static final String GRAPHITE="graphite"; + public static final String GANGLIA="ganglia"; + private String metricsAddress; + private String metricsHost; + private String metricsPrefix; + private String metricsServerName; + private int metricsPort; + private GraphiteReporter graphiteReporter = null; + private GangliaReporter gangliaReporter = null; + private long metricsReportIntervalSec; + private boolean isExternalMonitoringEnabled; + private static final TimeUnit RATE_UNIT = TimeUnit.MILLISECONDS; private static final TimeUnit DURATION_UNIT = TimeUnit.MILLISECONDS; + protected XLog LOG = XLog.getLog(getClass()); + /** * Creates the MetricsInstrumentation and starts taking some metrics. */ public MetricsInstrumentation() { metricRegistry = new MetricRegistry(); + isExternalMonitoringEnabled = ConfigurationService.getBoolean(EXTERNAL_MONITORING_ENABLE); + if(isExternalMonitoringEnabled) { + metricsServerName = ConfigurationService.get(EXTERNAL_MONITORING_TYPE); + if (metricsServerName != null) { + String modifiedServerName = metricsServerName.trim().toLowerCase(); + if (modifiedServerName.equals(GRAPHITE) || modifiedServerName.equals(GANGLIA)) { + metricsAddress = ConfigurationService.get(EXTERNAL_MONITORING_ADDRESS); + metricsPrefix = ConfigurationService.get(EXTERNAL_MONITORING_PREFIX); + metricsReportIntervalSec = ConfigurationService.getLong(EXTERNAL_MONITORING_INTERVAL); + LOG.debug("Publishing external monitoring to [{0}] at host [{1}] every [{2}] seconds with prefix " + + "[{3}]", metricsServerName, metricsAddress, metricsReportIntervalSec, metricsPrefix); + + try { + URL url = new URL(metricsAddress); + metricsHost = url.getHost(); + metricsPort = url.getPort(); + } catch (MalformedURLException e) { + LOG.error("Exception, ", e); + } + + if (modifiedServerName.equals(GRAPHITE)) { + Graphite graphite = new Graphite(new InetSocketAddress(metricsHost, metricsPort)); + graphiteReporter = GraphiteReporter.forRegistry(metricRegistry).prefixedWith(metricsPrefix) + .convertDurationsTo(TimeUnit.SECONDS).filter(MetricFilter.ALL).build(graphite); + graphiteReporter.start(metricsReportIntervalSec, TimeUnit.SECONDS); + } + + if (modifiedServerName.equals(GANGLIA)) { + GMetric ganglia; + try { + ganglia = new GMetric(metricsHost, metricsPort, GMetric.UDPAddressingMode.MULTICAST, 1); + } catch (IOException e) { + LOG.error("Exception, ", e); + throw new RuntimeException(e); + } + gangliaReporter = GangliaReporter.forRegistry(metricRegistry).prefixedWith(metricsPrefix) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(ganglia); + gangliaReporter.start(metricsReportIntervalSec, TimeUnit.SECONDS); + } + } else { + throw new RuntimeException("Metrics Server Name should be either graphite or ganglia"); + } + } + else { + throw new RuntimeException("Metrics Server Name is not specified"); + } + } + timersLock = new ReentrantLock(); gaugesLock = new ReentrantLock(); countersLock = new ReentrantLock(); @@ -116,6 +194,29 @@ public class MetricsInstrumentation extends Instrumentation { } /** + * Reporting final metrics into the server before stopping + */ + @Override + public void stop() { + if (graphiteReporter != null) { + try { + // reporting final metrics into graphite before stopping + graphiteReporter.report(); + } finally { + graphiteReporter.stop(); + } + } + if (gangliaReporter != null) { + try { + // reporting final metrics into ganglia before stopping + gangliaReporter.report(); + } finally { + gangliaReporter.stop(); + } + } + } + + /** * Add a cron to an instrumentation timer. The timer is created if it does not exists. <p> * Internally, this is backed by a {@link com.codahale.metrics.Timer}. * http://git-wip-us.apache.org/repos/asf/oozie/blob/67563df5/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 400569b..649691d 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -37,6 +37,39 @@ </property> <property> + <name>oozie.external_monitoring.enable</name> + <value>false</value> + <description> + If the oozie functional metrics needs to be exposed to the metrics-server backend, set it to true + If set to true, the following properties has to be specified : oozie.metrics.server.name, + oozie.metrics.host, oozie.metrics.prefix, oozie.metrics.report.interval.sec, oozie.metrics.port + </description> + </property> + + <property> + <name>oozie.external_monitoring.type</name> + <value>graphite</value> + <description> + The name of the server to which we want to send the metrics, would be graphite or ganglia. + </description> + </property> + + <property> + <name>oozie.external_monitoring.address</name> + <value>http://localhost:2020</value> + </property> + + <property> + <name>oozie.external_monitoring.metricPrefix</name> + <value>oozie</value> + </property> + + <property> + <name>oozie.external_monitoring.reporterIntervalSecs</name> + <value>60</value> + </property> + + <property> <name>oozie.action.mapreduce.uber.jar.enable</name> <value>false</value> <description> http://git-wip-us.apache.org/repos/asf/oozie/blob/67563df5/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java b/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java index 8dd78e9..67b4dc4 100644 --- a/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java +++ b/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java @@ -24,6 +24,8 @@ import com.codahale.metrics.Timer; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; + +import org.apache.oozie.service.Services; import org.apache.oozie.test.XTestCase; // Most tests adpated from TestInstrumentation @@ -38,6 +40,12 @@ public class TestMetricsInstrumentation extends XTestCase { } }; + @Override + protected void setUp() throws Exception { + super.setUp(); + new Services().init(); + } + public void testInstrumentationCounter() throws Exception { MetricsInstrumentation inst = new MetricsInstrumentation(); assertEquals(0, inst.getMetricRegistry().getCounters().size()); http://git-wip-us.apache.org/repos/asf/oozie/blob/67563df5/docs/src/site/twiki/AG_Install.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki index f043e7a..cd9f610 100644 --- a/docs/src/site/twiki/AG_Install.twiki +++ b/docs/src/site/twiki/AG_Install.twiki @@ -777,6 +777,43 @@ Once enabled, the =admin/instrumentation= REST endpoint will no longer be availa be used (see the [[WebServicesAPI#Oozie_Metrics][Web Services API]] documentation for more details); the Oozie Web UI will also replace the "Instrumentation" tab with a "Metrics" tab. +We can also publish the instrumentation metrics to the external server graphite or ganglia. For this the following +properties should be specified in oozie-site.xml : + <verbatim> + <property> + <name>oozie.external_monitoring.enable</name> + <value>false</value> + <description> + If the oozie functional metrics needs to be exposed to the metrics-server backend, set it to true + If set to true, the following properties has to be specified : oozie.metrics.server.name, + oozie.metrics.host, oozie.metrics.prefix, oozie.metrics.report.interval.sec, oozie.metrics.port + </description> + </property> + + <property> + <name>oozie.external_monitoring.type</name> + <value>graphite</value> + <description> + The name of the server to which we want to send the metrics, would be graphite or ganglia. + </description> + </property> + + <property> + <name>oozie.external_monitoring.address</name> + <value>http://localhost:2020</value> + </property> + + <property> + <name>oozie.external_monitoring.metricPrefix</name> + <value>oozie</value> + </property> + + <property> + <name>oozie.external_monitoring.reporterIntervalSecs</name> + <value>60</value> + </property> + </verbatim> + #HA ---+++ High Availability (HA) http://git-wip-us.apache.org/repos/asf/oozie/blob/67563df5/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 9e72bf2..a74ffab 100644 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,8 @@ <oozie.hadoop-utils.version>hadoop-${hadoop.majorversion}-${project.version}</oozie.hadoop-utils.version> <hbase.version>0.94.2</hbase.version> + <dropwizard.metrics.version>3.1.0</dropwizard.metrics.version> + <clover.license>/home/jenkins/tools/clover/latest/lib/clover.license</clover.license> <!-- Sharelib component versions --> @@ -1357,6 +1359,30 @@ </dependency> <dependency> + <groupId>info.ganglia.gmetric4j</groupId> + <artifactId>gmetric4j</artifactId> + <version>1.0.7</version> + </dependency> + + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-graphite</artifactId> + <version>${dropwizard.metrics.version}</version> + </dependency> + + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-ganglia</artifactId> + <version>${dropwizard.metrics.version}</version> + </dependency> + + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-core</artifactId> + <version>${dropwizard.metrics.version}</version> + </dependency> + + <dependency> <groupId>com.codahale.metrics</groupId> <artifactId>metrics-core</artifactId> <version>3.0.2</version> http://git-wip-us.apache.org/repos/asf/oozie/blob/67563df5/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 6d61c2d..7cbbcb5 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2251 Expose instrumental matrices in Realtime Graphing tool (nperiwal via rkanter) OOZIE-2314 Unable to kill old instance child job by workflow or coord rerun by Launcher (jaydeepvishwakarma via rkanter) OOZIE-2386 org.apache.oozie.util.TestXConfiguration.testSubstituteVar is flakey (rkanter) OOZIE-2385 org.apache.oozie.TestCoordinatorEngineStreamLog.testCoordLogStreaming is flakey (rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/67563df5/webapp/pom.xml ---------------------------------------------------------------------- diff --git a/webapp/pom.xml b/webapp/pom.xml index e42e219..7e73572 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -100,6 +100,11 @@ <groupId>org.apache.hive.hcatalog</groupId> <artifactId>hive-hcatalog-server-extensions</artifactId> </exclusion> + <!--Excluding this because of license problem--> + <exclusion> + <groupId>info.ganglia.gmetric4j</groupId> + <artifactId>gmetric4j</artifactId> + </exclusion> </exclusions> </dependency>
