Author: ddas
Date: Tue Sep 25 05:05:02 2007
New Revision: 579212
URL: http://svn.apache.org/viewvc?rev=579212&view=rev
Log:
HADOOP-1921. Save the configuration of completed/failed jobs and make them
available via the web-ui.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
lucene/hadoop/trunk/src/webapps/job/jobconf.jsp
lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp
lucene/hadoop/trunk/src/webapps/job/jobdetailshistory.jsp
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue Sep 25 05:05:02 2007
@@ -190,6 +190,9 @@
IMPROVEMENTS
+ HADOOP-1921. Save the configuration of completed/failed jobs and make them
+ available via the web-ui. (Amar Kamat via devaraj)
+
HADOOP-1266. Remove dependency of package org.apache.hadoop.net on
org.apache.hadoop.dfs. (Hairong Kuang via dhruba)
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java Tue
Sep 25 05:05:02 2007
@@ -316,23 +316,34 @@
public Map<String, Task> getAllTasks() { return allTasks; }
/**
+ * Get the path of the locally stored job file
+ * @param jobId id of the job
+ * @return the path of the job file on the local file system
+ */
+ public static String getLocalJobFilePath(String jobId){
+ return LOG_DIR + File.separator + jobId + "_conf.xml";
+ }
+
+ /**
* Log job submitted event to history. Creates a new file in history
* for the job. if history file creation fails, it disables history
* for all other events.
- * @param jobId job id assigned by job tracker.
- * @param jobName job name as given by user in job conf
- * @param user user name
+ * @param jobId job id assigned by job tracker.
+ * @param jobConf job conf of the job
+ * @param jobConfPath path to job conf xml file in HDFS.
* @param submitTime time when job tracker received the job
- * @param jobConf path to job conf xml file in HDFS.
*/
- public static void logSubmitted(String jobId, String jobName, String user,
- long submitTime, String jobConf){
-
+ public static void logSubmitted(String jobId, JobConf jobConf,
+ String jobConfPath, long submitTime) {
+ String jobName = jobConf.getJobName();
+ String user = jobConf.getUser();
if (!disableHistory){
synchronized(MASTER_INDEX_LOG_FILE){
JobHistory.log(masterIndex, RecordTypes.Job,
new Enum[]{Keys.JOBID, Keys.JOBNAME, Keys.USER,
Keys.SUBMIT_TIME, Keys.JOBCONF },
- new String[]{jobId, jobName, user,
String.valueOf(submitTime), jobConf });
+ new String[]{jobId, jobName, user,
+ String.valueOf(submitTime), jobConfPath}
+ );
}
// setup the history log file for this job
String logFileName = JOBTRACKER_START_TIME + "_" + jobId;
@@ -344,12 +355,27 @@
// add to writer as well
JobHistory.log(writer, RecordTypes.Job,
new Enum[]{Keys.JOBID, Keys.JOBNAME, Keys.USER,
Keys.SUBMIT_TIME, Keys.JOBCONF },
- new String[]{jobId, jobName, user,
String.valueOf(submitTime) , jobConf});
+ new String[]{jobId, jobName, user,
+ String.valueOf(submitTime) , jobConfPath}
+ );
}catch(IOException e){
LOG.error("Failed creating job history log file, disabling history",
e);
disableHistory = true;
}
+ }
+ /* Storing the job conf on the local file system */
+ String localJobFilePath = JobInfo.getLocalJobFilePath(jobId);
+ File localJobFile = new File(localJobFilePath);
+ try {
+ FileOutputStream jobOut = new FileOutputStream(localJobFile);
+ jobConf.write(jobOut);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Job conf for " + jobId + " stored at "
+ + localJobFile.getAbsolutePath());
+ }
+ } catch (IOException ioe) {
+ LOG.error("Failed to store job conf on the local filesystem ", ioe);
}
}
/**
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
Tue Sep 25 05:05:02 2007
@@ -159,9 +159,8 @@
this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent();
this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent();
- JobHistory.JobInfo.logSubmitted(jobid, conf.getJobName(), conf.getUser(),
- System.currentTimeMillis(),
- jobFile.toString());
+ JobHistory.JobInfo.logSubmitted(jobid, conf, jobFile.toString(),
+ System.currentTimeMillis());
MetricsContext metricsContext = MetricsUtil.getContext("mapred");
this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Tue
Sep 25 05:05:02 2007
@@ -1832,6 +1832,14 @@
}
}
+ /**
+ * Get the localized job file path on the job trackers local file system
+ * @param jobId id of the job
+ * @return the path of the job conf file on the local file system
+ */
+ public static String getLocalJobFilePath(String jobId){
+ return JobHistory.JobInfo.getLocalJobFilePath(jobId);
+ }
////////////////////////////////////////////////////////////
// main()
////////////////////////////////////////////////////////////
Modified: lucene/hadoop/trunk/src/webapps/job/jobconf.jsp
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/jobconf.jsp?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/jobconf.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/jobconf.jsp Tue Sep 25 05:05:02 2007
@@ -26,28 +26,12 @@
<h2>Job Configuration: JobId - <%= jobId %></h2><br>
<%
- JobInProgress job = (JobInProgress)tracker.getJob(jobId);
- if (job == null) {
- out.print("<h4>Job '" + jobId + "' not found!</h4><br>\n");
- return;
- }
-
- JobStatus status = job.getStatus();
- int runState = status.getRunState();
- if (runState != JobStatus.RUNNING) {
- out.print("<h4>Job '" + jobId + "' not running!</h4><br>\n");
- return;
- }
-
+ String jobFilePath = tracker.getLocalJobFilePath(jobId);
try {
- JobConf jobConf = job.getJobConf();
- ByteArrayOutputStream jobConfXML = new ByteArrayOutputStream();
- jobConf.write(jobConfXML);
+ JobConf jobConf = new JobConf(jobFilePath);
XMLUtils.transform(
jobConf.getConfResourceAsInputStream("webapps/static/jobconf.xsl"),
- new ByteArrayInputStream(jobConfXML.toByteArray()),
- out
- );
+ new FileInputStream(jobFilePath), out);
} catch (Exception e) {
out.println("Failed to retreive job configuration for job '" + jobId +
"!");
out.println(e);
Modified: lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp Tue Sep 25 05:05:02 2007
@@ -146,12 +146,8 @@
int flakyTaskTrackers = job.getNoOfBlackListedTrackers();
out.print("<b>User:</b> " + profile.getUser() + "<br>\n");
out.print("<b>Job Name:</b> " + profile.getJobName() + "<br>\n");
- if (runState == JobStatus.RUNNING) {
- out.print("<b>Job File:</b> <a href=\"jobconf.jsp?jobid=" + jobId +
"\">" +
- profile.getJobFile() + "</a><br>\n");
- } else {
- out.print("<b>Job File:</b> " + profile.getJobFile() + "<br>\n");
- }
+ out.print("<b>Job File:</b> <a href=\"jobconf.jsp?jobid=" + jobId + "\">"
+ + profile.getJobFile() + "</a><br>\n");
if (runState == JobStatus.RUNNING) {
out.print("<b>Status:</b> Running<br>\n");
out.print("<b>Started at:</b> " + new Date(job.getStartTime()) +
"<br>\n");
Modified: lucene/hadoop/trunk/src/webapps/job/jobdetailshistory.jsp
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/jobdetailshistory.jsp?rev=579212&r1=579211&r2=579212&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/jobdetailshistory.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/jobdetailshistory.jsp Tue Sep 25
05:05:02 2007
@@ -24,7 +24,8 @@
<b>User: </b> <%=job.get(Keys.USER) %><br/>
<b>JobName: </b> <%=job.get(Keys.JOBNAME) %><br/>
-<b>JobConf: </b> <%=job.get(Keys.JOBCONF) %><br/>
+<b>JobConf: </b> <a href="jobconf.jsp?jobid=<%=jobid %>">
+ <%=job.get(Keys.JOBCONF) %></a><br/>
<b>Submitted At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getLong(Keys.SUBMIT_TIME), 0 ) %><br/>
<b>Launched At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getLong(Keys.LAUNCH_TIME), job.getLong(Keys.SUBMIT_TIME)) %><br/>
<b>Finished At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat,
job.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME)) %><br/>
