Author: cutting
Date: Tue Apr 7 20:39:30 2009
New Revision: 762947
URL: http://svn.apache.org/viewvc?rev=762947&view=rev
Log:
HADOOP-5469. Add /metrics servlet to daemons, providing metrics over HTTP as
either text or JSON. Contributed by Philip Zeyliger.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java
hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java
hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java
hadoop/core/trunk/src/saveVersion.sh
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Apr 7 20:39:30 2009
@@ -80,6 +80,9 @@
HADOOP-5518. Add contrib/mrunit, a MapReduce unit test framework.
(Aaron Kimball via cutting)
+ HADOOP-5469. Add /metrics servlet to daemons, providing metrics
+ over HTTP as either text or JSON. (Philip Zeyliger via cutting)
+
IMPROVEMENTS
HADOOP-4565. Added CombineFileInputFormat to use data locality information
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java
Tue Apr 7 20:39:30 2009
@@ -42,7 +42,7 @@
* @return the date in unix 'date' format
*/
String date();
-
+
/**
* Get the url for the subversion repository.
*/
@@ -59,4 +59,11 @@
* @return The branch name, e.g. "trunk" or "branches/branch-0.20"
*/
String branch();
+
+ /**
+ * Get a checksum of the source files from which
+ * Hadoop was compiled.
+ * @return a string that uniquely identifies the source
+ **/
+ String srcChecksum();
}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java Tue Apr
7 20:39:30 2009
@@ -37,6 +37,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.log.LogLevel;
+import org.apache.hadoop.metrics.MetricsServlet;
import org.apache.hadoop.util.ReflectionUtils;
import org.mortbay.jetty.Connector;
@@ -188,6 +189,7 @@
// set up default servlets
addServlet("stacks", "/stacks", StackServlet.class);
addServlet("logLevel", "/logLevel", LogLevel.Servlet.class);
+ addServlet("metrics", "/metrics", MetricsServlet.class);
}
public void addContext(Context ctxt, boolean isFiltered)
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java
Tue Apr 7 20:39:30 2009
@@ -22,6 +22,8 @@
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -142,6 +144,14 @@
IllegalAccessException {
return getContext(contextName, contextName);
}
+
+ /**
+ * Returns all MetricsContexts built by this factory.
+ */
+ public synchronized Collection<MetricsContext> getAllContexts() {
+ // Make a copy to avoid race conditions with creating new contexts.
+ return new ArrayList<MetricsContext>(contextMap.values());
+ }
/**
* Returns a "null" context - one which does nothing.
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java
Tue Apr 7 20:39:30 2009
@@ -21,6 +21,10 @@
package org.apache.hadoop.metrics;
import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.hadoop.metrics.spi.OutputRecord;
/**
* The main interface to the metrics package.
@@ -103,5 +107,12 @@
* Returns the timer period.
*/
public abstract int getPeriod();
-
+
+ /**
+ * Retrieves all the records managed by this MetricsContext.
+ * Useful for monitoring systems that are polling-based.
+ *
+ * @return A non-null map from all record names to the records managed.
+ */
+ Map<String, Collection<OutputRecord>> getAllRecords();
}
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java
Tue Apr 7 20:39:30 2009
@@ -27,7 +27,6 @@
import java.io.PrintWriter;
import org.apache.hadoop.metrics.ContextFactory;
-import org.apache.hadoop.metrics.MetricsException;
import org.apache.hadoop.metrics.spi.AbstractMetricsContext;
import org.apache.hadoop.metrics.spi.OutputRecord;
@@ -62,18 +61,7 @@
file = new File(fileName);
}
- String periodStr = getAttribute(PERIOD_PROPERTY);
- if (periodStr != null) {
- int period = 0;
- try {
- period = Integer.parseInt(periodStr);
- } catch (NumberFormatException nfe) {
- }
- if (period <= 0) {
- throw new MetricsException("Invalid period: " + periodStr);
- }
- setPeriod(period);
- }
+ parseAndSetPeriod(PERIOD_PROPERTY);
}
/**
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
---
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java
(original)
+++
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java
Tue Apr 7 20:39:30 2009
@@ -88,19 +88,7 @@
public void init(String contextName, ContextFactory factory) {
super.init(contextName, factory);
-
- String periodStr = getAttribute(PERIOD_PROPERTY);
- if (periodStr != null) {
- int period = 0;
- try {
- period = Integer.parseInt(periodStr);
- } catch (NumberFormatException nfe) {
- }
- if (period <= 0) {
- throw new MetricsException("Invalid period: " + periodStr);
- }
- setPeriod(period);
- }
+ parseAndSetPeriod(PERIOD_PROPERTY);
metricsServers =
Util.parse(getAttribute(SERVERS_PROPERTY), DEFAULT_PORT);
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
---
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
(original)
+++
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
Tue Apr 7 20:39:30 2009
@@ -26,6 +26,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
@@ -60,7 +61,7 @@
private ContextFactory factory = null;
private String contextName = null;
- static class TagMap extends TreeMap<String,Object> {
+ public static class TagMap extends TreeMap<String,Object> {
private static final long serialVersionUID = 3546309335061952993L;
TagMap() {
super();
@@ -83,8 +84,14 @@
}
}
- static class MetricMap extends TreeMap<String,Number> {
+ public static class MetricMap extends TreeMap<String,Number> {
private static final long serialVersionUID = -7495051861141631609L;
+ MetricMap() {
+ super();
+ }
+ MetricMap(MetricMap orig) {
+ super(orig);
+ }
}
static class RecordMap extends HashMap<TagMap,MetricMap> {
@@ -309,6 +316,28 @@
}
flush();
}
+
+ /**
+ * Retrieves all the records managed by this MetricsContext.
+ * Useful for monitoring systems that are polling-based.
+ * @return A non-null collection of all monitoring records.
+ */
+ public synchronized Map<String, Collection<OutputRecord>> getAllRecords() {
+ Map<String, Collection<OutputRecord>> out = new TreeMap<String,
Collection<OutputRecord>>();
+ for (String recordName : bufferedData.keySet()) {
+ RecordMap recordMap = bufferedData.get(recordName);
+ synchronized (recordMap) {
+ List<OutputRecord> records = new ArrayList<OutputRecord>();
+ Set<Entry<TagMap, MetricMap>> entrySet = recordMap.entrySet();
+ for (Entry<TagMap, MetricMap> entry : entrySet) {
+ OutputRecord outRec = new OutputRecord(entry.getKey(),
entry.getValue());
+ records.add(outRec);
+ }
+ out.put(recordName, records);
+ }
+ }
+ return out;
+ }
/**
* Sends a record to the metrics system.
@@ -424,4 +453,23 @@
protected void setPeriod(int period) {
this.period = period;
}
+
+ /**
+ * If a period is set in the attribute passed in, override
+ * the default with it.
+ */
+ protected void parseAndSetPeriod(String attributeName) {
+ String periodStr = getAttribute(attributeName);
+ if (periodStr != null) {
+ int period = 0;
+ try {
+ period = Integer.parseInt(periodStr);
+ } catch (NumberFormatException nfe) {
+ }
+ if (period <= 0) {
+ throw new MetricsException("Invalid period: " + periodStr);
+ }
+ setPeriod(period);
+ }
+ }
}
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
---
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java
(original)
+++
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java
Tue Apr 7 20:39:30 2009
@@ -44,21 +44,7 @@
public void init(String contextName, ContextFactory factory) {
super.init(contextName, factory);
-
- // If period is specified, use it, otherwise the default is good enough
-
- String periodStr = getAttribute(PERIOD_PROPERTY);
- if (periodStr != null) {
- int period = 0;
- try {
- period = Integer.parseInt(periodStr);
- } catch (NumberFormatException nfe) {
- }
- if (period <= 0) {
- throw new MetricsException("Invalid period: " + periodStr);
- }
- setPeriod(period);
- }
+ parseAndSetPeriod(PERIOD_PROPERTY);
}
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java
Tue Apr 7 20:39:30 2009
@@ -21,7 +21,11 @@
package org.apache.hadoop.metrics.spi;
import java.util.Collections;
+import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+
import org.apache.hadoop.metrics.spi.AbstractMetricsContext.MetricMap;
import org.apache.hadoop.metrics.spi.AbstractMetricsContext.TagMap;
@@ -68,5 +72,19 @@
public Number getMetric(String name) {
return metricMap.get(name);
}
-
+
+
+ /**
+ * Returns a copy of this record's tags.
+ */
+ public TagMap getTagsCopy() {
+ return new TagMap(tagMap);
+ }
+
+ /**
+ * Returns a copy of this record's metrics.
+ */
+ public MetricMap getMetricsCopy() {
+ return new MetricMap(metricMap);
+ }
}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java Tue Apr
7 20:39:30 2009
@@ -87,7 +87,15 @@
public static String getUrl() {
return version != null ? version.url() : "Unknown";
}
-
+
+ /**
+ * Get the checksum of the source files from which Hadoop was
+ * built.
+ **/
+ public static String getSrcChecksum() {
+ return version != null ? version.srcChecksum() : "Unknown";
+ }
+
/**
* Returns the buildVersion which includes version,
* revision, user and date.
@@ -96,12 +104,13 @@
return VersionInfo.getVersion() +
" from " + VersionInfo.getRevision() +
" by " + VersionInfo.getUser() +
- " on " + VersionInfo.getDate();
+ " source checksum " + VersionInfo.getSrcChecksum();
}
public static void main(String[] args) {
System.out.println("Hadoop " + getVersion());
System.out.println("Subversion " + getUrl() + " -r " + getRevision());
System.out.println("Compiled by " + getUser() + " on " + getDate());
+ System.out.println("From source with checksum " + getSrcChecksum());
}
}
Modified: hadoop/core/trunk/src/saveVersion.sh
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/saveVersion.sh?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/saveVersion.sh (original)
+++ hadoop/core/trunk/src/saveVersion.sh Tue Apr 7 20:39:30 2009
@@ -41,16 +41,19 @@
branch="Unknown"
url="file://$cwd"
fi
+srcChecksum=`find src -name '*.java' | LC_ALL=C sort | xargs md5sum | md5sum |
cut -d ' ' -f 1`
+
mkdir -p build/src/org/apache/hadoop
cat << EOF | \
sed -e "s/VERSION/$version/" -e "s/USER/$user/" -e "s/DATE/$date/" \
-e "s|URL|$url|" -e "s/REV/$revision/" \
- -e "s|BRANCH|$branch|" \
+ -e "s|BRANCH|$branch|" -e "s/SRCCHECKSUM/$srcChecksum/" \
> build/src/org/apache/hadoop/package-info.java
/*
* Generated by src/saveVersion.sh
*/
@HadoopVersionAnnotation(version="VERSION", revision="REV", branch="BRANCH",
- user="USER", date="DATE", url="URL")
+ user="USER", date="DATE", url="URL",
+ srcChecksum="SRCCHECKSUM")
package org.apache.hadoop;
EOF
