hadoop git commit: MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state. Contributed by zhihai xu. (

2015-10-30 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 0af4f222a -> 530b8a75c


MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists
to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED
state. Contributed by zhihai xu.
(cherry picked from commit 1ea90144d58443a7431ff33ba49ee19278ebe42b)

Conflicts:

hadoop-mapreduce-project/CHANGES.txt

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/530b8a75
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/530b8a75
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/530b8a75

Branch: refs/heads/branch-2.6
Commit: 530b8a75cb3a8a8c122a6026603d4070ea7b7ec4
Parents: 0af4f22
Author: Jason Lowe 
Authored: Fri Oct 30 15:15:49 2015 +
Committer: Jason Lowe 
Committed: Fri Oct 30 15:15:49 2015 +

--
 hadoop-mapreduce-project/CHANGES.txt|  4 +++
 .../mapreduce/v2/hs/HistoryFileManager.java | 11 +++---
 .../mapreduce/v2/hs/TestHistoryFileManager.java | 35 
 3 files changed, 46 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/530b8a75/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt 
b/hadoop-mapreduce-project/CHANGES.txt
index 53f938f..ce1e065 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -12,6 +12,10 @@ Release 2.6.3 - UNRELEASED
 
   BUG FIXES
 
+MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists 
to 
+avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
+(zhihai xu via devaraj)
+
 Release 2.6.2 - 2015-10-21
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/530b8a75/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 2f74d15..d0e171e 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -299,8 +299,9 @@ public class HistoryFileManager extends AbstractService {
 private JobIndexInfo jobIndexInfo;
 private HistoryInfoState state;
 
-private HistoryFileInfo(Path historyFile, Path confFile, Path summaryFile,
-JobIndexInfo jobIndexInfo, boolean isInDone) {
+@VisibleForTesting
+protected HistoryFileInfo(Path historyFile, Path confFile,
+Path summaryFile, JobIndexInfo jobIndexInfo, boolean isInDone) {
   this.historyFile = historyFile;
   this.confFile = confFile;
   this.summaryFile = summaryFile;
@@ -333,7 +334,8 @@ public class HistoryFileManager extends AbstractService {
  + " historyFile = " + historyFile;
 }
 
-private synchronized void moveToDone() throws IOException {
+@VisibleForTesting
+synchronized void moveToDone() throws IOException {
   if (LOG.isDebugEnabled()) {
 LOG.debug("moveToDone: " + historyFile);
   }
@@ -364,7 +366,8 @@ public class HistoryFileManager extends AbstractService {
   paths.add(confFile);
 }
 
-if (summaryFile == null) {
+if (summaryFile == null || !intermediateDoneDirFc.util().exists(
+summaryFile)) {
   LOG.info("No summary file for job: " + jobId);
 } else {
   String jobSummaryString = getJobSummary(intermediateDoneDirFc,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/530b8a75/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java

hadoop git commit: MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state. Contributed by zhihai xu.

2015-05-15 Thread devaraj 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 d9455c790 - b9865cbf3


MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists
to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED
state. Contributed by zhihai xu.

(cherry picked from commit 1ea90144d58443a7431ff33ba49ee19278ebe42b)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9865cbf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9865cbf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9865cbf

Branch: refs/heads/branch-2
Commit: b9865cbf3d435903835afa23c74c5e2ecfbba1bc
Parents: d9455c7
Author: Devaraj K deva...@apache.org
Authored: Fri May 15 12:31:32 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri May 15 12:34:08 2015 +0530

--
 hadoop-mapreduce-project/CHANGES.txt|  4 +++
 .../mapreduce/v2/hs/HistoryFileManager.java | 11 +++---
 .../mapreduce/v2/hs/TestHistoryFileManager.java | 35 
 3 files changed, 46 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9865cbf/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt 
b/hadoop-mapreduce-project/CHANGES.txt
index fd4cd72..9d943bd 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -167,6 +167,10 @@ Release 2.8.0 - UNRELEASED
 MAPREDUCE-5708. Duplicate String.format in 
YarnOutputFiles.getSpillFileForWrite.
 (Konstantin Weitz via devaraj)
 
+MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists 
to 
+avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
+(zhihai xu via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9865cbf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 6420303..0b9eaee 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -303,8 +303,9 @@ public class HistoryFileManager extends AbstractService {
 private JobIndexInfo jobIndexInfo;
 private HistoryInfoState state;
 
-private HistoryFileInfo(Path historyFile, Path confFile, Path summaryFile,
-JobIndexInfo jobIndexInfo, boolean isInDone) {
+@VisibleForTesting
+protected HistoryFileInfo(Path historyFile, Path confFile,
+Path summaryFile, JobIndexInfo jobIndexInfo, boolean isInDone) {
   this.historyFile = historyFile;
   this.confFile = confFile;
   this.summaryFile = summaryFile;
@@ -337,7 +338,8 @@ public class HistoryFileManager extends AbstractService {
  +  historyFile =  + historyFile;
 }
 
-private synchronized void moveToDone() throws IOException {
+@VisibleForTesting
+synchronized void moveToDone() throws IOException {
   if (LOG.isDebugEnabled()) {
 LOG.debug(moveToDone:  + historyFile);
   }
@@ -368,7 +370,8 @@ public class HistoryFileManager extends AbstractService {
   paths.add(confFile);
 }
 
-if (summaryFile == null) {
+if (summaryFile == null || !intermediateDoneDirFc.util().exists(
+summaryFile)) {
   LOG.info(No summary file for job:  + jobId);
 } else {
   String jobSummaryString = getJobSummary(intermediateDoneDirFc,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9865cbf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index 1c5cc5c..7694de9 100644
---

hadoop git commit: MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state. Contributed by zhihai xu.

2015-05-15 Thread devaraj 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 3bef7c80a - 1ea90144d


MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists
to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED
state. Contributed by zhihai xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ea90144
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ea90144
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ea90144

Branch: refs/heads/trunk
Commit: 1ea90144d58443a7431ff33ba49ee19278ebe42b
Parents: 3bef7c8
Author: Devaraj K deva...@apache.org
Authored: Fri May 15 12:31:32 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri May 15 12:31:32 2015 +0530

--
 hadoop-mapreduce-project/CHANGES.txt|  4 +++
 .../mapreduce/v2/hs/HistoryFileManager.java | 11 +++---
 .../mapreduce/v2/hs/TestHistoryFileManager.java | 35 
 3 files changed, 46 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea90144/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt 
b/hadoop-mapreduce-project/CHANGES.txt
index bc1f427..7e34297 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -425,6 +425,10 @@ Release 2.8.0 - UNRELEASED
 MAPREDUCE-5708. Duplicate String.format in 
YarnOutputFiles.getSpillFileForWrite.
 (Konstantin Weitz via devaraj)
 
+MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists 
to 
+avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
+(zhihai xu via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea90144/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 69f814d..f0786da 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -303,8 +303,9 @@ public class HistoryFileManager extends AbstractService {
 private JobIndexInfo jobIndexInfo;
 private HistoryInfoState state;
 
-private HistoryFileInfo(Path historyFile, Path confFile, Path summaryFile,
-JobIndexInfo jobIndexInfo, boolean isInDone) {
+@VisibleForTesting
+protected HistoryFileInfo(Path historyFile, Path confFile,
+Path summaryFile, JobIndexInfo jobIndexInfo, boolean isInDone) {
   this.historyFile = historyFile;
   this.confFile = confFile;
   this.summaryFile = summaryFile;
@@ -337,7 +338,8 @@ public class HistoryFileManager extends AbstractService {
  +  historyFile =  + historyFile;
 }
 
-private synchronized void moveToDone() throws IOException {
+@VisibleForTesting
+synchronized void moveToDone() throws IOException {
   if (LOG.isDebugEnabled()) {
 LOG.debug(moveToDone:  + historyFile);
   }
@@ -368,7 +370,8 @@ public class HistoryFileManager extends AbstractService {
   paths.add(confFile);
 }
 
-if (summaryFile == null) {
+if (summaryFile == null || !intermediateDoneDirFc.util().exists(
+summaryFile)) {
   LOG.info(No summary file for job:  + jobId);
 } else {
   String jobSummaryString = getJobSummary(intermediateDoneDirFc,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea90144/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index 1c5cc5c..7694de9 100644
---