Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 0af4f222a -> 530b8a75c
MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists
to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED
state. Contributed by zhihai xu.
(cherry picked from commit 1ea90144d58443a7431ff33ba49ee19278ebe42b)
Conflicts:
hadoop-mapreduce-project/CHANGES.txt
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/530b8a75
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/530b8a75
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/530b8a75
Branch: refs/heads/branch-2.6
Commit: 530b8a75cb3a8a8c122a6026603d4070ea7b7ec4
Parents: 0af4f22
Author: Jason Lowe <[email protected]>
Authored: Fri Oct 30 15:15:49 2015 +0000
Committer: Jason Lowe <[email protected]>
Committed: Fri Oct 30 15:15:49 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 4 +++
.../mapreduce/v2/hs/HistoryFileManager.java | 11 +++---
.../mapreduce/v2/hs/TestHistoryFileManager.java | 35 ++++++++++++++++++++
3 files changed, 46 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/530b8a75/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 53f938f..ce1e065 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -12,6 +12,10 @@ Release 2.6.3 - UNRELEASED
BUG FIXES
+ MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists
to
+ avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
+ (zhihai xu via devaraj)
+
Release 2.6.2 - 2015-10-21
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/530b8a75/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 2f74d15..d0e171e 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -299,8 +299,9 @@ public class HistoryFileManager extends AbstractService {
private JobIndexInfo jobIndexInfo;
private HistoryInfoState state;
- private HistoryFileInfo(Path historyFile, Path confFile, Path summaryFile,
- JobIndexInfo jobIndexInfo, boolean isInDone) {
+ @VisibleForTesting
+ protected HistoryFileInfo(Path historyFile, Path confFile,
+ Path summaryFile, JobIndexInfo jobIndexInfo, boolean isInDone) {
this.historyFile = historyFile;
this.confFile = confFile;
this.summaryFile = summaryFile;
@@ -333,7 +334,8 @@ public class HistoryFileManager extends AbstractService {
+ " historyFile = " + historyFile;
}
- private synchronized void moveToDone() throws IOException {
+ @VisibleForTesting
+ synchronized void moveToDone() throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("moveToDone: " + historyFile);
}
@@ -364,7 +366,8 @@ public class HistoryFileManager extends AbstractService {
paths.add(confFile);
}
- if (summaryFile == null) {
+ if (summaryFile == null || !intermediateDoneDirFc.util().exists(
+ summaryFile)) {
LOG.info("No summary file for job: " + jobId);
} else {
String jobSummaryString = getJobSummary(intermediateDoneDirFc,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/530b8a75/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
----------------------------------------------------------------------
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index e2e943a..e5b38ff 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
@@ -31,7 +31,12 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.test.CoreTestDriver;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -199,4 +204,34 @@ public class TestHistoryFileManager {
testCreateHistoryDirs(dfsCluster.getConfiguration(0), clock);
}
+ @Test
+ public void testHistoryFileInfoSummaryFileNotExist() throws Exception {
+ HistoryFileManagerTest hmTest = new HistoryFileManagerTest();
+ String job = "job_1410889000000_123456";
+ Path summaryFile = new Path(job + ".summary");
+ JobIndexInfo jobIndexInfo = new JobIndexInfo();
+ jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(job)));
+ Configuration conf = dfsCluster.getConfiguration(0);
+ conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR,
+ "/" + UUID.randomUUID());
+ conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
+ "/" + UUID.randomUUID());
+ hmTest.serviceInit(conf);
+ HistoryFileInfo info = hmTest.getHistoryFileInfo(null, null,
+ summaryFile, jobIndexInfo, false);
+ info.moveToDone();
+ Assert.assertFalse(info.didMoveFail());
+ }
+
+ static class HistoryFileManagerTest extends HistoryFileManager {
+ public HistoryFileManagerTest() {
+ super();
+ }
+ public HistoryFileInfo getHistoryFileInfo(Path historyFile,
+ Path confFile, Path summaryFile, JobIndexInfo jobIndexInfo,
+ boolean isInDone) {
+ return new HistoryFileInfo(historyFile, confFile, summaryFile,
+ jobIndexInfo, isInDone);
+ }
+ }
}
