Author: jing9
Date: Fri Feb 14 18:32:37 2014
New Revision: 1568437
URL: http://svn.apache.org/r1568437
Log:
Merging r1566359 through r1568420 from trunk.
Added:
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/lib/TestCombineFileRecordReader.java
- copied unchanged from r1568420,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/lib/TestCombineFileRecordReader.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileRecordReader.java
- copied unchanged from r1568420,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileRecordReader.java
Modified:
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
Propchange: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1566359-1568420
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt Fri
Feb 14 18:32:37 2014
@@ -158,8 +158,26 @@ Release 2.4.0 - UNRELEASED
OPTIMIZATIONS
BUG FIXES
+
+ MAPREDUCE-5746. Job diagnostics can implicate wrong task for a failed job.
+ (Jason Lowe via kasha)
+
+ MAPREDUCE-5670. CombineFileRecordReader should report progress when moving
+ to the next file (Chen He via jlowe)
+
+Release 2.3.1 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
-Release 2.3.0 - UNRELEASED
+Release 2.3.0 - 2014-02-18
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1565517-1568420
Modified:
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java
Fri Feb 14 18:32:37 2014
@@ -140,6 +140,8 @@ public class CombineFileRecordReader<K,
return false;
}
+ reporter.progress();
+
// get a record reader for the idx-th chunk
try {
curReader = rrConstructor.newInstance(new Object []
Modified:
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
Fri Feb 14 18:32:37 2014
@@ -353,8 +353,10 @@ public class JobHistoryParser implements
taskInfo.error = StringInterner.weakIntern(event.getError());
taskInfo.failedDueToAttemptId = event.getFailedAttemptID();
taskInfo.counters = event.getCounters();
- info.errorInfo = "Task " + taskInfo.taskId +" failed " +
- taskInfo.attemptsMap.size() + " times ";
+ if (info.errorInfo.isEmpty()) {
+ info.errorInfo = "Task " + taskInfo.taskId + " failed " +
+ taskInfo.attemptsMap.size() + " times ";
+ }
}
private void handleTaskStartedEvent(TaskStartedEvent event) {
Modified:
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
Fri Feb 14 18:32:37 2014
@@ -54,7 +54,7 @@ public class CombineFileRecordReader<K,
protected int idx;
protected long progress;
protected RecordReader<K, V> curReader;
-
+
public void initialize(InputSplit split,
TaskAttemptContext context) throws IOException, InterruptedException {
this.split = (CombineFileSplit)split;
@@ -144,6 +144,8 @@ public class CombineFileRecordReader<K,
return false;
}
+ context.progress();
+
// get a record reader for the idx-th chunk
try {
Configuration conf = context.getConfiguration();
Modified:
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
Fri Feb 14 18:32:37 2014
@@ -40,6 +40,8 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TypeConverter;
@@ -51,7 +53,9 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
@@ -69,7 +73,6 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo;
import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo;
-import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.net.DNSToSwitchMapping;
@@ -730,4 +733,40 @@ public class TestJobHistoryParsing {
assertNull(test.getAMInfos());
}
+
+ @Test
+ public void testMultipleFailedTasks() throws Exception {
+ JobHistoryParser parser =
+ new JobHistoryParser(Mockito.mock(FSDataInputStream.class));
+ EventReader reader = Mockito.mock(EventReader.class);
+ final AtomicInteger numEventsRead = new AtomicInteger(0); // Hack!
+ final org.apache.hadoop.mapreduce.TaskType taskType =
+ org.apache.hadoop.mapreduce.TaskType.MAP;
+ final TaskID[] tids = new TaskID[2];
+ JobID jid = new JobID("1", 1);
+ tids[0] = new TaskID(jid, taskType, 0);
+ tids[1] = new TaskID(jid, taskType, 1);
+ Mockito.when(reader.getNextEvent()).thenAnswer(
+ new Answer<HistoryEvent>() {
+ public HistoryEvent answer(InvocationOnMock invocation)
+ throws IOException {
+ // send two task start and two task fail events for tasks 0 and 1
+ int eventId = numEventsRead.getAndIncrement();
+ TaskID tid = tids[eventId & 0x1];
+ if (eventId < 2) {
+ return new TaskStartedEvent(tid, 0, taskType, "");
+ }
+ if (eventId < 4) {
+ TaskFailedEvent tfe = new TaskFailedEvent(tid, 0, taskType,
+ "failed", "FAILED", null, new Counters());
+ tfe.setDatum(tfe.getDatum());
+ return tfe;
+ }
+ return null;
+ }
+ });
+ JobInfo info = parser.parse(reader);
+ assertTrue("Task 0 not implicated",
+ info.getErrorInfo().contains(tids[0].toString()));
+ }
}
