Author: omalley
Date: Tue Mar 8 06:01:10 2011
New Revision: 1079258
URL: http://svn.apache.org/viewvc?rev=1079258&view=rev
Log:
commit 83220625a2489cdc52b5256a006f6775b3502be3
Author: Greg Roelofs <[email protected]>
Date: Tue Feb 1 20:05:35 2011 -0800
Uber-specific job counters to help with QA automated testing (ArunR
request).
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1079258&r1=1079257&r2=1079258&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
Tue Mar 8 06:01:10 2011
@@ -1776,7 +1776,8 @@ public class JobInProgress {
} else {
++runningReduceTasks;
name = TaskType.REDUCE;
- counter = JobCounter.TOTAL_LAUNCHED_REDUCES;
+ counter = tip.isUberTask()? JobCounter.TOTAL_LAUNCHED_UBERTASKS
+ : JobCounter.TOTAL_LAUNCHED_REDUCES;
if (tip.isSpeculating()) {
speculativeReduceTasks++;
metrics.speculateReduce(id);
@@ -3408,6 +3409,8 @@ public class JobInProgress {
if (!tip.isJobCleanupTask() && !tip.isJobSetupTask()) {
if (tip.isMapTask()) {
jobCounters.incrCounter(JobCounter.NUM_FAILED_MAPS, 1);
+ } else if (tip.isUberTask()) {
+ jobCounters.incrCounter(JobCounter.NUM_FAILED_UBERTASKS, 1);
} else {
jobCounters.incrCounter(JobCounter.NUM_FAILED_REDUCES, 1);
}
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java?rev=1079258&r1=1079257&r2=1079258&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
Tue Mar 8 06:01:10 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapred.Task.TaskReporter;
import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.MRJobConfig; // JobContext.SKIP_RECORDS
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex;
import org.apache.hadoop.mapreduce.TaskType; // MAP, JOB_SETUP,
TASK_CLEANUP...
@@ -413,7 +414,12 @@ class UberTask extends Task {
* completed
*/
private void updateCounters(Task subtask) {
- getCounters().incrAllCounters(subtask.getCounters());
+ Counters counters = getCounters();
+ if (counters != null) {
+ counters.incrCounter(subtask.isMapTask()?
+ JobCounter.NUM_UBER_SUBMAPS : JobCounter.NUM_UBER_SUBREDUCES, 1);
+ counters.incrAllCounters(subtask.getCounters());
+ }
}
@Override
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java?rev=1079258&r1=1079257&r2=1079258&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java
Tue Mar 8 06:01:10 2011
@@ -27,8 +27,12 @@ import org.apache.hadoop.classification.
public enum JobCounter {
NUM_FAILED_MAPS,
NUM_FAILED_REDUCES,
+ NUM_FAILED_UBERTASKS,
TOTAL_LAUNCHED_MAPS,
TOTAL_LAUNCHED_REDUCES,
+ TOTAL_LAUNCHED_UBERTASKS,
+ NUM_UBER_SUBMAPS,
+ NUM_UBER_SUBREDUCES,
OTHER_LOCAL_MAPS,
DATA_LOCAL_MAPS,
RACK_LOCAL_MAPS,
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties?rev=1079258&r1=1079257&r2=1079258&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties
Tue Mar 8 06:01:10 2011
@@ -16,8 +16,12 @@ CounterGroupName= Job C
NUM_FAILED_MAPS.name= Failed map tasks
NUM_FAILED_REDUCES.name= Failed reduce tasks
+NUM_FAILED_UBERTASKS.name= Failed UberTasks
TOTAL_LAUNCHED_MAPS.name= Launched map tasks
TOTAL_LAUNCHED_REDUCES.name= Launched reduce tasks
+TOTAL_LAUNCHED_UBERTASKS.name= Launched UberTasks (as reduce tasks)
+NUM_UBER_SUBMAPS.name= Completed UberTask sub-maps
+NUM_UBER_SUBREDUCES.name= Completed UberTask sub-reduces
OTHER_LOCAL_MAPS.name= Other local map tasks
DATA_LOCAL_MAPS.name= Data-local map tasks
RACK_LOCAL_MAPS.name= Rack-local map tasks
