Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1079193&r1=1079192&r2=1079193&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java Tue Mar 8 05:54:13 2011 @@ -302,6 +302,9 @@ public class JobHistoryParser { info.launchTime = event.getLaunchTime(); info.totalMaps = event.getTotalMaps(); info.totalReduces = event.getTotalReduces(); + info.isUber = event.getIsUber(); + info.numUberSubMaps = event.getNumUberSubMaps(); + info.numUberSubReduces = event.getNumUberSubReduces(); } private void handleJobInfoChangeEvent(JobInfoChangeEvent event) { @@ -334,6 +337,9 @@ public class JobHistoryParser { long launchTime; int totalMaps; int totalReduces; + boolean isUber; + int numUberSubMaps; + int numUberSubReduces; int failedMaps; int failedReduces; int finishedMaps; @@ -352,8 +358,9 @@ public class JobHistoryParser { */ public JobInfo() { submitTime = launchTime = finishTime = -1; - totalMaps = totalReduces = failedMaps = failedReduces = 0; - finishedMaps = finishedReduces = 0; + isUber = false; + totalMaps = totalReduces = numUberSubMaps = numUberSubReduces = 0; + failedMaps = failedReduces = finishedMaps = finishedReduces = 0; username = jobname = jobConfPath = jobQueueName = ""; tasksMap = new HashMap<TaskID, TaskInfo>(); jobACLs = new HashMap<JobACL, AccessControlList>(); @@ -370,6 +377,7 @@ public class JobHistoryParser { System.out.println("PRIORITY: " + priority); System.out.println("TOTAL_MAPS: " + totalMaps); System.out.println("TOTAL_REDUCES: " + totalReduces); + //GRR FIXME: add UBER_SUBMAPS and UBER_SUBREDUCES? (or only if isUber == true? coordinate with TaskInfo printAll() changes) System.out.println("MAP_COUNTERS:" + mapCounters.toString()); System.out.println("REDUCE_COUNTERS:" + reduceCounters.toString()); System.out.println("TOTAL_COUNTERS: " + totalCounters.toString()); @@ -395,18 +403,24 @@ public class JobHistoryParser { public String getJobConfPath() { return jobConfPath; } /** Get the job launch time */ public long getLaunchTime() { return launchTime; } - /** Get the total number of maps */ - public long getTotalMaps() { return totalMaps; } - /** Get the total number of reduces */ - public long getTotalReduces() { return totalReduces; } + /** Get the total number of "real" maps */ + public int getTotalMaps() { return totalMaps; } + /** Get the total number of "real" reduces */ + public int getTotalReduces() { return totalReduces; } + /** Was the job small enough to be converted to an UberTask? */ + public boolean getIsUber() { return isUber; } + /** Get the number of sub-MapTasks within the UberTask */ + public int getNumUberSubMaps() { return numUberSubMaps; } + /** Get the number of sub-ReduceTasks within the UberTask */ + public int getNumUberSubReduces() { return numUberSubReduces; } /** Get the total number of failed maps */ - public long getFailedMaps() { return failedMaps; } + public int getFailedMaps() { return failedMaps; } /** Get the number of failed reduces */ - public long getFailedReduces() { return failedReduces; } + public int getFailedReduces() { return failedReduces; } /** Get the number of finished maps */ - public long getFinishedMaps() { return finishedMaps; } + public int getFinishedMaps() { return finishedMaps; } /** Get the number of finished reduces */ - public long getFinishedReduces() { return finishedReduces; } + public int getFinishedReduces() { return finishedReduces; } /** Get the job status */ public String getJobStatus() { return jobStatus; } public String getErrorInfo() { return errorInfo; }
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java?rev=1079193&r1=1079192&r2=1079193&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java Tue Mar 8 05:54:13 2011 @@ -44,11 +44,15 @@ public class JobInitedEvent implements H * @param jobStatus */ public JobInitedEvent(JobID id, long launchTime, int totalMaps, - int totalReduces, String jobStatus) { + int totalReduces, boolean isUber, int numUberSubMaps, + int numUberSubReduces, String jobStatus) { datum.jobid = new Utf8(id.toString()); datum.launchTime = launchTime; datum.totalMaps = totalMaps; datum.totalReduces = totalReduces; + datum.isUber = isUber; + datum.numUberSubMaps = numUberSubMaps; + datum.numUberSubReduces = numUberSubReduces; datum.jobStatus = new Utf8(jobStatus); } @@ -61,10 +65,16 @@ public class JobInitedEvent implements H public JobID getJobId() { return JobID.forName(datum.jobid.toString()); } /** Get the launch time */ public long getLaunchTime() { return datum.launchTime; } - /** Get the total number of maps */ + /** Get the total number of "real" maps */ public int getTotalMaps() { return datum.totalMaps; } - /** Get the total number of reduces */ + /** Get the total number of "real" reduces */ public int getTotalReduces() { return datum.totalReduces; } + /** Was the job small enough to be converted to an UberTask? */ + public boolean getIsUber() { return datum.isUber; } + /** Get the number of sub-MapTasks within the UberTask */ + public int getNumUberSubMaps() { return datum.numUberSubMaps; } + /** Get the number of sub-ReduceTasks within the UberTask */ + public int getNumUberSubReduces() { return datum.numUberSubReduces; } /** Get the status */ public String getStatus() { return datum.jobStatus.toString(); } /** Get the event type */ Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java?rev=1079193&r1=1079192&r2=1079193&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java Tue Mar 8 05:54:13 2011 @@ -241,8 +241,12 @@ public class TestJobQueueTaskScheduler e } @Override + public boolean getIsUber() { + return t.isUberTask(); + } + + @Override public void addFetchFailedMap(TaskAttemptID mapTaskId) { - } }; status.setRunState(TaskStatus.State.RUNNING); Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java?rev=1079193&r1=1079192&r2=1079193&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java Tue Mar 8 05:54:13 2011 @@ -103,7 +103,7 @@ public class TestTaskStatus { // check the default case String test = "hi"; final int maxSize = 16; - TaskStatus status = new TaskStatus(null, 0, 0, null, test, test, null, null, + TaskStatus status = new TaskStatus(null, 0, 0, null, test, test, null, null, null) { @Override protected int getMaxStringSize() { @@ -118,6 +118,11 @@ public class TestTaskStatus { public boolean getIsMap() { return false; } + + @Override + public boolean getIsUber() { + return false; + } }; assertEquals("Small diagnostic info test failed", status.getDiagnosticInfo(), test); @@ -198,6 +203,11 @@ public class TestTaskStatus { public boolean getIsMap() { return false; } + + @Override + public boolean getIsUber() { + return false; + } }; assertEquals("Large diagnostic info test failed", maxSize, status.getDiagnosticInfo().length()); Modified: hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java?rev=1079193&r1=1079192&r2=1079193&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java Tue Mar 8 05:54:13 2011 @@ -125,10 +125,15 @@ public class Job20LineHistoryEventEmitte String status = line.get("JOB_STATUS"); String totalMaps = line.get("TOTAL_MAPS"); String totalReduces = line.get("TOTAL_REDUCES"); + // note: UberTask playback not supported since uber data not yet logged + boolean isUber = false; + int numUberSubMaps = 0; + int numUberSubReduces = 0; if (launchTime != null && totalMaps != null && totalReduces != null) { - return new JobInitedEvent(jobID, Long.parseLong(launchTime), Integer - .parseInt(totalMaps), Integer.parseInt(totalReduces), status); + return new JobInitedEvent(jobID, Long.parseLong(launchTime), + Integer.parseInt(totalMaps), Integer.parseInt(totalReduces), + isUber, numUberSubMaps, numUberSubReduces, status); } return null; Modified: hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp?rev=1079193&r1=1079192&r2=1079193&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp Tue Mar 8 05:54:13 2011 @@ -365,10 +365,16 @@ "<th>Killed</th>" + "<th><a href=\"jobfailures.jsp?jobid=" + jobId + "\">Failed/Killed<br>Task Attempts</a></th></tr>\n"); - printTaskSummary(out, jobId, "map", status.mapProgress(), - job.getTasks(TaskType.MAP)); - printTaskSummary(out, jobId, "reduce", status.reduceProgress(), - job.getTasks(TaskType.REDUCE)); + if (job.getUberMode()) { + /* placeholder until true task- and attempt-level uber info available */ + printTaskSummary(out, jobId, "uber", status.reduceProgress(), + job.getTasks(TaskType.REDUCE)); + } else { + printTaskSummary(out, jobId, "map", status.mapProgress(), + job.getTasks(TaskType.MAP)); + printTaskSummary(out, jobId, "reduce", status.reduceProgress(), + job.getTasks(TaskType.REDUCE)); + } out.print("</table>\n"); %> @@ -421,6 +427,7 @@ %> </table> +<%if (job.getTasks(TaskType.MAP).length > 0) { %> <hr>Map Completion Graph - <% if("off".equals(request.getParameter("map.graph"))) { @@ -442,10 +449,14 @@ if("off".equals(session.getAttribute("ma width="<%=TaskGraphServlet.width + 2 * TaskGraphServlet.xmargin%>" height="<%=TaskGraphServlet.height + 3 * TaskGraphServlet.ymargin%>" style="width:100%" type="image/svg+xml" pluginspage="http://www.adobe.com/svg/viewer/install/"; /> -<%}%> +<%} }%> -<%if(job.getTasks(TaskType.REDUCE).length > 0) { %> +<%if (job.getTasks(TaskType.REDUCE).length > 0) { %> +<%if (job.getUberMode()) { %> +<hr>UberTask Completion Graph - +<%} else { %> <hr>Reduce Completion Graph - +<%}%> <%if("off".equals(session.getAttribute("reduce.graph"))) { %> <a href="/jobdetails.jsp?jobid=<%=jobId%>&refresh=<%=refresh%>&reduce.graph=on" > open </a> <%} else { %> Modified: hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp?rev=1079193&r1=1079192&r2=1079193&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp Tue Mar 8 05:54:13 2011 @@ -56,6 +56,12 @@ if (job == null) { return; } + // MR-1220 FIXME (LATER): to fully integrate uberization, need task- + // and attempt-level JobHistory/Avro/etc. changes to multiple Event + // types; instead, going with top-level hack for now + boolean isUber = job.getIsUber(); + int numUberSubMaps = job.getNumUberSubMaps(); + int numUberSubReduces = job.getNumUberSubReduces(); if (job.getJobStatus().equals("FAILED")) reasonforFailure = job.getErrorInfo(); %> @@ -90,7 +96,7 @@ <center> <table border="2" cellpadding="5" cellspacing="2"> <tr> -<td>Kind</td><td>Total Tasks(successful+failed+killed)</td><td>Successful tasks</td><td>Failed tasks</td><td>Killed tasks</td><td>Start Time</td><td>Finish Time</td> +<td>Kind</td><td>Total Tasks (successful+<wbr>failed+<wbr>killed)</td><td>Successful tasks</td><td>Failed tasks</td><td>Killed tasks</td><td>Start Time</td><td>Finish Time</td> </tr> <tr> <td>Setup</td> @@ -106,6 +112,10 @@ <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getSetupFinished(), sj.getSetupStarted()) %></td> </tr> <tr> + +<% + if (!isUber) { +%> <td>Map</td> <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=MAP&status=all"> <%=sj.getTotalMaps()%></a></td> @@ -131,6 +141,36 @@ <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceStarted(), 0) %></td> <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceFinished(), sj.getReduceStarted()) %></td> </tr> + +<% + } else /* isUber */ { +%> + +<tr> +<td>Uber</td> + <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=all"> + <%=sj.getTotalReduces()%></a></td> + <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=SUCCEEDED"> + <%=job.getFinishedReduces()%></a></td> + <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=FAILED"> + <%=sj.getNumFailedReduces()%></a></td> + <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=KILLED"> + <%=sj.getNumKilledReduces()%></a></td> + <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceStarted(), 0) %></td> + <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceFinished(), sj.getReduceStarted()) %></td> +</tr> +<td>Map subtasks</td> + <td colspan="6"><%=numUberSubMaps%></td> +</tr> +<tr> +<td>Reduce subtasks</td> + <td colspan="6"><%=numUberSubReduces%></td> +</tr> + +<% + } +%> + <tr> <td>Cleanup</td> <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=JOB_CLEANUP&status=all">
