Author: amareshwari
Date: Mon Aug 16 06:22:39 2010
New Revision: 985803
URL: http://svn.apache.org/viewvc?rev=985803&view=rev
Log:
MAPREDUCE-1980. Merge -r 985797:985798 from trunk
Modified:
hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java
Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=985803&r1=985802&r2=985803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Mon Aug 16 06:22:39 2010
@@ -1509,3 +1509,7 @@ Release 0.21.0 - 2010-08-13
MAPREDUCE-2012. Some contrib tests fail in branch 0.21 and trunk.
(Amareshwari Sriramadasu via tomwhite)
+ MAPREDUCE-1980. Fixes TaskAttemptUnsuccessfulCompletionEvent and
+ TaskAttemptFinishedEvent to correctly log event type for all task types.
+ (Amar Kamat via amareshwari)
+
Modified:
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java?rev=985803&r1=985802&r2=985803&view=diff
==============================================================================
---
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
(original)
+++
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
Mon Aug 16 06:22:39 2010
@@ -92,7 +92,11 @@ public class TaskAttemptFinishedEvent i
Counters getCounters() { return EventReader.fromAvro(datum.counters); }
/** Get the event type */
public EventType getEventType() {
- return EventType.MAP_ATTEMPT_FINISHED;
+ // Note that the task type can be setup/map/reduce/cleanup but the
+ // attempt-type can only be map/reduce.
+ return getTaskId().getTaskType() == TaskType.MAP
+ ? EventType.MAP_ATTEMPT_FINISHED
+ : EventType.REDUCE_ATTEMPT_FINISHED;
}
}
Modified:
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java?rev=985803&r1=985802&r2=985803&view=diff
==============================================================================
---
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
(original)
+++
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
Mon Aug 16 06:22:39 2010
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapred.TaskStatus;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
@@ -87,7 +88,17 @@ public class TaskAttemptUnsuccessfulComp
public String getTaskStatus() { return datum.status.toString(); }
/** Get the event type */
public EventType getEventType() {
- return EventType.MAP_ATTEMPT_KILLED;
+ // Note that the task type can be setup/map/reduce/cleanup but the
+ // attempt-type can only be map/reduce.
+ // find out if the task failed or got killed
+ boolean failed =
TaskStatus.State.FAILED.toString().equals(getTaskStatus());
+ return getTaskId().getTaskType() == TaskType.MAP
+ ? (failed
+ ? EventType.MAP_ATTEMPT_FAILED
+ : EventType.MAP_ATTEMPT_KILLED)
+ : (failed
+ ? EventType.REDUCE_ATTEMPT_FAILED
+ : EventType.REDUCE_ATTEMPT_KILLED);
}
}
Modified:
hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java?rev=985803&r1=985802&r2=985803&view=diff
==============================================================================
---
hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java
(original)
+++
hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java
Mon Aug 16 06:22:39 2010
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.mapreduce.jobhistory;
+import org.apache.hadoop.mapred.TaskStatus;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskType;
@@ -27,38 +29,124 @@ import junit.framework.TestCase;
*/
public class TestJobHistoryEvents extends TestCase {
/**
- * Test TaskAttemptStartedEvent.
+ * Test {...@link TaskAttemptStartedEvent} for various task types.
+ */
+ private static void testAttemptStartedEventForTypes(EventType expected,
+ TaskAttemptID id,
+ TaskType[] types) {
+ for (TaskType t : types) {
+ TaskAttemptStartedEvent tase =
+ new TaskAttemptStartedEvent(id, t, 0L, "", 0);
+ assertEquals(expected, tase.getEventType());
+ }
+ }
+
+ /**
+ * Test {...@link TaskAttemptStartedEvent}.
*/
public void testTaskAttemptStartedEvent() {
EventType expected = EventType.MAP_ATTEMPT_STARTED;
TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1);
- // check jobsetup type
- TaskAttemptStartedEvent tase =
- new TaskAttemptStartedEvent(fakeId, TaskType.JOB_SETUP, 0L, "", 0);
- assertEquals(expected, tase.getEventType());
-
- // check jobcleanup type
- tase = new TaskAttemptStartedEvent(fakeId, TaskType.JOB_CLEANUP, 0L, "",
0);
- assertEquals(expected, tase.getEventType());
-
- // check map type
- tase = new TaskAttemptStartedEvent(fakeId, TaskType.MAP, 0L, "", 0);
- assertEquals(expected, tase.getEventType());
+ // check the events for job-setup, job-cleanup and map task-types
+ testAttemptStartedEventForTypes(expected, fakeId,
+ new TaskType[] {TaskType.JOB_SETUP,
+ TaskType.JOB_CLEANUP,
+ TaskType.MAP});
expected = EventType.REDUCE_ATTEMPT_STARTED;
fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1);
- // check jobsetup type
- tase = new TaskAttemptStartedEvent(fakeId, TaskType.JOB_SETUP, 0L, "", 0);
- assertEquals(expected, tase.getEventType());
-
- // check jobcleanup type
- tase = new TaskAttemptStartedEvent(fakeId, TaskType.JOB_CLEANUP, 0L, "",
0);
- assertEquals(expected, tase.getEventType());
-
- // check reduce type
- tase = new TaskAttemptStartedEvent(fakeId, TaskType.REDUCE, 0L, "", 0);
- assertEquals(expected, tase.getEventType());
+ // check the events for job-setup, job-cleanup and reduce task-types
+ testAttemptStartedEventForTypes(expected, fakeId,
+ new TaskType[] {TaskType.JOB_SETUP,
+ TaskType.JOB_CLEANUP,
+ TaskType.REDUCE});
+ }
+
+ /**
+ * Test {...@link TaskAttemptUnsuccessfulCompletionEvent} for various task
types.
+ */
+ private static void testFailedKilledEventsForTypes(EventType expected,
+ TaskAttemptID id,
+ TaskType[] types,
+ String state) {
+ for (TaskType t : types) {
+ TaskAttemptUnsuccessfulCompletionEvent tauce =
+ new TaskAttemptUnsuccessfulCompletionEvent(id, t, state, 0L, "", "");
+ assertEquals(expected, tauce.getEventType());
+ }
+ }
+
+ /**
+ * Test {...@link TaskAttemptUnsuccessfulCompletionEvent} for killed/failed
task.
+ */
+ public void testTaskAttemptUnsuccessfulCompletionEvent() {
+ TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1);
+
+ // check killed events for job-setup, job-cleanup and map task-types
+ testFailedKilledEventsForTypes(EventType.MAP_ATTEMPT_KILLED, fakeId,
+ new TaskType[] {TaskType.JOB_SETUP,
+ TaskType.JOB_CLEANUP,
+ TaskType.MAP},
+ TaskStatus.State.KILLED.toString());
+ // check failed events for job-setup, job-cleanup and map task-types
+ testFailedKilledEventsForTypes(EventType.MAP_ATTEMPT_FAILED, fakeId,
+ new TaskType[] {TaskType.JOB_SETUP,
+ TaskType.JOB_CLEANUP,
+ TaskType.MAP},
+ TaskStatus.State.FAILED.toString());
+
+ fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1);
+
+ // check killed events for job-setup, job-cleanup and reduce task-types
+ testFailedKilledEventsForTypes(EventType.REDUCE_ATTEMPT_KILLED, fakeId,
+ new TaskType[] {TaskType.JOB_SETUP,
+ TaskType.JOB_CLEANUP,
+ TaskType.REDUCE},
+ TaskStatus.State.KILLED.toString());
+ // check failed events for job-setup, job-cleanup and reduce task-types
+ testFailedKilledEventsForTypes(EventType.REDUCE_ATTEMPT_FAILED, fakeId,
+ new TaskType[] {TaskType.JOB_SETUP,
+ TaskType.JOB_CLEANUP,
+ TaskType.REDUCE},
+ TaskStatus.State.FAILED.toString());
+ }
+
+ /**
+ * Test {...@link TaskAttemptFinishedEvent} for various task types.
+ */
+ private static void testFinishedEventsForTypes(EventType expected,
+ TaskAttemptID id,
+ TaskType[] types) {
+ for (TaskType t : types) {
+ TaskAttemptFinishedEvent tafe =
+ new TaskAttemptFinishedEvent(id, t,
+ TaskStatus.State.SUCCEEDED.toString(), 0L, "", "", new Counters());
+ assertEquals(expected, tafe.getEventType());
+ }
+ }
+
+ /**
+ * Test {...@link TaskAttemptFinishedEvent} for finished task.
+ */
+ public void testTaskAttemptFinishedEvent() {
+ EventType expected = EventType.MAP_ATTEMPT_FINISHED;
+ TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1);
+
+ // check the events for job-setup, job-cleanup and map task-types
+ testFinishedEventsForTypes(expected, fakeId,
+ new TaskType[] {TaskType.JOB_SETUP,
+ TaskType.JOB_CLEANUP,
+ TaskType.MAP});
+
+ expected = EventType.REDUCE_ATTEMPT_FINISHED;
+ fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1);
+
+ // check the events for job-setup, job-cleanup and reduce task-types
+ testFinishedEventsForTypes(expected, fakeId,
+ new TaskType[] {TaskType.JOB_SETUP,
+ TaskType.JOB_CLEANUP,
+ TaskType.REDUCE});
}
}
