MAPREDUCE-6252. JobHistoryServer should not fail when encountering a missing directory. Contributed by Craig Welch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e67c4d3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e67c4d3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e67c4d3 Branch: refs/heads/HDFS-7240 Commit: 5e67c4d384193b38a85655c8f93193596821faa5 Parents: 618ba70 Author: Devaraj K <[email protected]> Authored: Mon Apr 27 15:01:42 2015 +0530 Committer: Devaraj K <[email protected]> Committed: Mon Apr 27 15:01:42 2015 +0530 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../mapreduce/v2/hs/HistoryFileManager.java | 19 ++++++++++------- .../mapreduce/v2/hs/TestHistoryFileManager.java | 22 ++++++++++++++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e67c4d3/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 4166e6a..f9488fb 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -349,6 +349,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are broken due to MAPREDUCE-6297. (Siqi Li via gera) + MAPREDUCE-6252. JobHistoryServer should not fail when encountering a + missing directory. (Craig Welch via devaraj) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e67c4d3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 65f8a4f..69f814d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -740,17 +740,22 @@ public class HistoryFileManager extends AbstractService { } } - private static List<FileStatus> scanDirectory(Path path, FileContext fc, + @VisibleForTesting + protected static List<FileStatus> scanDirectory(Path path, FileContext fc, PathFilter pathFilter) throws IOException { path = fc.makeQualified(path); List<FileStatus> jhStatusList = new ArrayList<FileStatus>(); - RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path); - while (fileStatusIter.hasNext()) { - FileStatus fileStatus = fileStatusIter.next(); - Path filePath = fileStatus.getPath(); - if (fileStatus.isFile() && pathFilter.accept(filePath)) { - jhStatusList.add(fileStatus); + try { + RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path); + while (fileStatusIter.hasNext()) { + FileStatus fileStatus = fileStatusIter.next(); + Path filePath = fileStatus.getPath(); + if (fileStatus.isFile() && pathFilter.accept(filePath)) { + jhStatusList.add(fileStatus); + } } + } catch (FileNotFoundException fe) { + LOG.error("Error while scanning directory " + path, fe); } return jhStatusList; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e67c4d3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java index e2e943a..1c5cc5c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java @@ -21,13 +21,17 @@ package org.apache.hadoop.mapreduce.v2.hs; import java.io.File; import java.io.FileOutputStream; +import java.io.FileNotFoundException; import java.util.UUID; +import java.util.List; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -45,6 +49,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; +import static org.mockito.Mockito.*; + public class TestHistoryFileManager { private static MiniDFSCluster dfsCluster = null; private static MiniDFSCluster dfsCluster2 = null; @@ -199,4 +205,20 @@ public class TestHistoryFileManager { testCreateHistoryDirs(dfsCluster.getConfiguration(0), clock); } + @Test + public void testScanDirectory() throws Exception { + + Path p = new Path("any"); + FileContext fc = mock(FileContext.class); + when(fc.makeQualified(p)).thenReturn(p); + when(fc.listStatus(p)).thenThrow(new FileNotFoundException()); + + List<FileStatus> lfs = HistoryFileManager.scanDirectory(p, fc, null); + + //primarily, succcess is that an exception was not thrown. Also nice to + //check this + Assert.assertNotNull(lfs); + + } + }
