Author: cutting
Date: Fri Jun 22 12:09:12 2007
New Revision: 549933
URL: http://svn.apache.org/viewvc?view=rev&rev=549933
Log:
HADOOP-1518. Add a session id to job metrics, for use by HOD. Contributed by
David Bowen.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Jun 22 12:09:12 2007
@@ -239,6 +239,9 @@
73. HADOOP-1512. Fix failing TestTextInputFormat on Windows.
(Senthil Subramanian via nigel)
+ 74. HADOOP-1518. Add a session id to job metrics, for use by HOD.
+ (David Bowen via cutting)
+
Release 0.13.0 - 2007-06-08
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Fri Jun
22 12:09:12 2007
@@ -597,6 +597,34 @@
}
/**
+ * Get the user-specified session identifier. The default is the empty
string.
+ *
+ * The session identifier is used to tag metric data that is reported to some
+ * performance metrics system via the org.apache.hadoop.metrics API. The
+ * session identifier is intended, in particular, for use by
Hadoop-On-Demand
+ * (HOD) which allocates a virtual Hadoop cluster dynamically and
transiently.
+ * HOD will set the session identifier by modifying the hadoop-site.xml file
+ * before starting the cluster.
+ *
+ * When not running under HOD, this identifer is expected to remain set to
+ * the empty string.
+ *
+ * @return the session identifier, defaulting to ""
+ */
+ public String getSessionId() {
+ return get("session.id", "");
+ }
+
+ /**
+ * Set the user-specified session idengifier.
+ *
+ * @param sessionId the new session id
+ */
+ public void setSessionId(String sessionId) {
+ set("session.id", sessionId);
+ }
+
+ /**
* Set the maximum no. of failures of a given job per tasktracker.
*
* @param noFailures maximum no. of failures of a given job per tasktracker.
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
Fri Jun 22 12:09:12 2007
@@ -152,8 +152,9 @@
System.currentTimeMillis(), jobFile);
MetricsContext metricsContext = MetricsUtil.getContext("mapred");
- this.jobMetrics = metricsContext.createRecord("job");
+ this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");
this.jobMetrics.setTag("user", conf.getUser());
+ this.jobMetrics.setTag("sessionId", conf.getSessionId());
this.jobMetrics.setTag("jobName", conf.getJobName());
this.jobMetrics.setTag("jobId", jobid);
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri
Jun 22 12:09:12 2007
@@ -424,9 +424,10 @@
private int numJobsSubmitted = 0;
private int numJobsCompleted = 0;
- JobTrackerMetrics() {
+ JobTrackerMetrics(JobConf conf) {
MetricsContext context = MetricsUtil.getContext("mapred");
metricsRecord = MetricsUtil.createRecord(context, "jobtracker");
+ metricsRecord.setTag("sessionId", conf.getSessionId());
context.registerUpdater(this);
}
@@ -651,7 +652,7 @@
this.startTime = System.currentTimeMillis();
- myMetrics = new JobTrackerMetrics();
+ myMetrics = new JobTrackerMetrics(jobConf);
this.expireTrackersThread = new Thread(this.expireTrackers,
"expireTrackers");
this.expireTrackersThread.start();
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
Fri Jun 22 12:09:12 2007
@@ -253,7 +253,7 @@
public LocalJobRunner(Configuration conf) throws IOException {
this.fs = FileSystem.get(conf);
this.conf = conf;
- myMetrics = new JobTrackerMetrics();
+ myMetrics = new JobTrackerMetrics(new JobConf(conf));
}
// JobSubmissionProtocol methods
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java Fri
Jun 22 12:09:12 2007
@@ -747,6 +747,7 @@
this.shuffleMetrics =
MetricsUtil.createRecord(metricsContext, "shuffleInput");
this.shuffleMetrics.setTag("user", conf.getUser());
+ this.shuffleMetrics.setTag("sessionId", conf.getSessionId());
// Seed the random number generator with a reasonably globally unique
seed
long randomSeed = System.nanoTime() +
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Fri
Jun 22 12:09:12 2007
@@ -162,8 +162,10 @@
private int numCompletedTasks = 0;
TaskTrackerMetrics() {
+ JobConf conf = getJobConf();
MetricsContext context = MetricsUtil.getContext("mapred");
metricsRecord = MetricsUtil.createRecord(context, "tasktracker");
+ metricsRecord.setTag("sessionId", conf.getSessionId());
context.registerUpdater(this);
}
@@ -317,10 +319,10 @@
int numCopiers = this.fConf.getInt("mapred.reduce.parallel.copies", 5);
//tweak the probe sample size (make it a function of numCopiers)
probe_sample_size = Math.max(numCopiers*5, 50);
-
-
+
+
this.myMetrics = new TaskTrackerMetrics();
-
+
// port numbers
this.taskReportPort = this.fConf.getInt("mapred.task.tracker.report.port",
50050);
// bind address
