Repository: hadoop Updated Branches: refs/heads/branch-2 fad8c7817 -> 01c0bcb17 refs/heads/branch-2.7 2742f12b5 -> 8e1c33e70 refs/heads/trunk 972f1f1ab -> 2c238ae4e
HDFS-7917. Use file to replace data dirs in test to simulate a disk failure. Contributed by Lei (Eddy) Xu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c238ae4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c238ae4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c238ae4 Branch: refs/heads/trunk Commit: 2c238ae4e00371ef76582b007bb0e20ac8455d9c Parents: 972f1f1 Author: cnauroth <[email protected]> Authored: Mon Mar 23 16:29:51 2015 -0700 Committer: cnauroth <[email protected]> Committed: Mon Mar 23 16:29:51 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/datanode/DataNodeTestUtils.java | 61 +++++++++++++++++++- .../datanode/TestDataNodeHotSwapVolumes.java | 29 ++++------ .../datanode/TestDataNodeVolumeFailure.java | 11 +--- .../TestDataNodeVolumeFailureReporting.java | 46 ++++----------- .../TestDataNodeVolumeFailureToleration.java | 8 +-- 6 files changed, 88 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c238ae4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8c99876..b88b7e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -774,6 +774,9 @@ Release 2.7.0 - UNRELEASED HDFS-7962. Remove duplicated logs in BlockManager. (yliu) + HDFS-7917. Use file to replace data dirs in test to simulate a disk failure. + (Lei (Eddy) Xu via cnauroth) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c238ae4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java index fd51e52..f9a2ba1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java @@ -40,7 +40,9 @@ import com.google.common.base.Preconditions; * Utility class for accessing package-private DataNode information during tests. * */ -public class DataNodeTestUtils { +public class DataNodeTestUtils { + private static final String DIR_FAILURE_SUFFIX = ".origin"; + public static DatanodeRegistration getDNRegistrationForBP(DataNode dn, String bpid) throws IOException { return dn.getDNRegistrationForBP(bpid); @@ -159,4 +161,61 @@ public class DataNodeTestUtils { final String bpid, final long blkId) { return FsDatasetTestUtil.fetchReplicaInfo(dn.getFSDataset(), bpid, blkId); } + + /** + * It injects disk failures to data dirs by replacing these data dirs with + * regular files. + * + * @param dirs data directories. + * @throws IOException on I/O error. + */ + public static void injectDataDirFailure(File... dirs) throws IOException { + for (File dir : dirs) { + File renamedTo = new File(dir.getPath() + DIR_FAILURE_SUFFIX); + if (renamedTo.exists()) { + throw new IOException(String.format( + "Can not inject failure to dir: %s because %s exists.", + dir, renamedTo)); + } + if (!dir.renameTo(renamedTo)) { + throw new IOException(String.format("Failed to rename %s to %s.", + dir, renamedTo)); + } + if (!dir.createNewFile()) { + throw new IOException(String.format( + "Failed to create file %s to inject disk failure.", dir)); + } + } + } + + /** + * Restore the injected data dir failures. + * + * @see {@link #injectDataDirFailures}. + * @param dirs data directories. + * @throws IOException + */ + public static void restoreDataDirFromFailure(File... dirs) + throws IOException { + for (File dir : dirs) { + File renamedDir = new File(dir.getPath() + DIR_FAILURE_SUFFIX); + if (renamedDir.exists()) { + if (dir.exists()) { + if (!dir.isFile()) { + throw new IOException( + "Injected failure data dir is supposed to be file: " + dir); + } + if (!dir.delete()) { + throw new IOException( + "Failed to delete injected failure data dir: " + dir); + } + } + if (!renamedDir.renameTo(dir)) { + throw new IOException(String.format( + "Failed to recover injected failure data dir %s to %s.", + renamedDir, dir)); + } + } + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c238ae4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 8ab3dd2..2f51d45 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.BlockMissingException; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -682,26 +681,18 @@ public class TestDataNodeHotSwapVolumes { failedVolume != null); long used = failedVolume.getDfsUsed(); - try { - assertTrue("Couldn't chmod local vol: " + dirToFail, - FileUtil.setExecutable(dirToFail, false)); - // Call and wait DataNode to detect disk failure. - long lastDiskErrorCheck = dn.getLastDiskErrorCheck(); - dn.checkDiskErrorAsync(); - while (dn.getLastDiskErrorCheck() == lastDiskErrorCheck) { - Thread.sleep(100); - } - - createFile(new Path("/test1"), 32, (short)2); - assertEquals(used, failedVolume.getDfsUsed()); - } finally { - // Need to restore the mode on dirToFail. Otherwise, if an Exception - // is thrown above, the following tests can not delete this data directory - // and thus fail to start MiniDFSCluster. - assertTrue("Couldn't restore executable for: " + dirToFail, - FileUtil.setExecutable(dirToFail, true)); + DataNodeTestUtils.injectDataDirFailure(dirToFail); + // Call and wait DataNode to detect disk failure. + long lastDiskErrorCheck = dn.getLastDiskErrorCheck(); + dn.checkDiskErrorAsync(); + while (dn.getLastDiskErrorCheck() == lastDiskErrorCheck) { + Thread.sleep(100); } + createFile(new Path("/test1"), 32, (short)2); + assertEquals(used, failedVolume.getDfsUsed()); + + DataNodeTestUtils.restoreDataDirFromFailure(dirToFail); dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir); createFile(new Path("/test2"), 32, (short)2); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c238ae4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java index 9cbad6d..0428b81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java @@ -121,10 +121,6 @@ public class TestDataNodeVolumeFailure { if(cluster != null) { cluster.shutdown(); } - for (int i = 0; i < 3; i++) { - FileUtil.setExecutable(new File(dataDir, "data"+(2*i+1)), true); - FileUtil.setExecutable(new File(dataDir, "data"+(2*i+2)), true); - } } /* @@ -159,7 +155,7 @@ public class TestDataNodeVolumeFailure { !deteteBlocks(failedDir) ) { throw new IOException("Could not delete hdfs directory '" + failedDir + "'"); - } + } data_fail.setReadOnly(); failedDir.setReadOnly(); System.out.println("Deleteing " + failedDir.getPath() + "; exist=" + failedDir.exists()); @@ -217,7 +213,7 @@ public class TestDataNodeVolumeFailure { DFSTestUtil.waitReplication(fs, file1, (short) 2); File dn0Vol1 = new File(dataDir, "data" + (2 * 0 + 1)); - assertTrue(FileUtil.setExecutable(dn0Vol1, false)); + DataNodeTestUtils.injectDataDirFailure(dn0Vol1); DataNode dn0 = cluster.getDataNodes().get(0); long lastDiskErrorCheck = dn0.getLastDiskErrorCheck(); dn0.checkDiskErrorAsync(); @@ -291,8 +287,7 @@ public class TestDataNodeVolumeFailure { // Fail the first volume on both datanodes File dn1Vol1 = new File(dataDir, "data"+(2*0+1)); File dn2Vol1 = new File(dataDir, "data"+(2*1+1)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, false)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, false)); + DataNodeTestUtils.injectDataDirFailure(dn1Vol1, dn2Vol1); Path file2 = new Path("/test2"); DFSTestUtil.createFile(fs, file2, 1024, (short)3, 1L); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c238ae4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java index 9842f25..aac288a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java @@ -34,7 +34,6 @@ import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.ReconfigurationException; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; @@ -87,19 +86,6 @@ public class TestDataNodeVolumeFailureReporting { @After public void tearDown() throws Exception { - // Restore executable permission on all directories where a failure may have - // been simulated by denying execute access. This is based on the maximum - // number of datanodes and the maximum number of storages per data node used - // throughout the tests in this suite. - assumeTrue(!Path.WINDOWS); - int maxDataNodes = 3; - int maxStoragesPerDataNode = 4; - for (int i = 0; i < maxDataNodes; i++) { - for (int j = 1; j <= maxStoragesPerDataNode; j++) { - String subDir = "data" + ((i * maxStoragesPerDataNode) + j); - FileUtil.setExecutable(new File(dataDir, subDir), true); - } - } IOUtils.cleanup(LOG, fs); if (cluster != null) { cluster.shutdown(); @@ -141,8 +127,7 @@ public class TestDataNodeVolumeFailureReporting { * fail. The client does not retry failed nodes even though * perhaps they could succeed because just a single volume failed. */ - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, false)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, false)); + DataNodeTestUtils.injectDataDirFailure(dn1Vol1, dn2Vol1); /* * Create file1 and wait for 3 replicas (ie all DNs can still @@ -179,7 +164,7 @@ public class TestDataNodeVolumeFailureReporting { * Now fail a volume on the third datanode. We should be able to get * three replicas since we've already identified the other failures. */ - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn3Vol1, false)); + DataNodeTestUtils.injectDataDirFailure(dn3Vol1); Path file2 = new Path("/test2"); DFSTestUtil.createFile(fs, file2, 1024, (short)3, 1L); DFSTestUtil.waitReplication(fs, file2, (short)3); @@ -208,7 +193,7 @@ public class TestDataNodeVolumeFailureReporting { * and that it's no longer up. Only wait for two replicas since * we'll never get a third. */ - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn3Vol2, false)); + DataNodeTestUtils.injectDataDirFailure(dn3Vol2); Path file3 = new Path("/test3"); DFSTestUtil.createFile(fs, file3, 1024, (short)3, 1L); DFSTestUtil.waitReplication(fs, file3, (short)2); @@ -233,10 +218,8 @@ public class TestDataNodeVolumeFailureReporting { * restart, so file creation should be able to succeed after * restoring the data directories and restarting the datanodes. */ - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, true)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, true)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn3Vol1, true)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn3Vol2, true)); + DataNodeTestUtils.restoreDataDirFromFailure( + dn1Vol1, dn2Vol1, dn3Vol1, dn3Vol2); cluster.restartDataNodes(); cluster.waitActive(); Path file4 = new Path("/test4"); @@ -275,8 +258,7 @@ public class TestDataNodeVolumeFailureReporting { // third healthy so one node in the pipeline will not fail). File dn1Vol1 = new File(dataDir, "data"+(2*0+1)); File dn2Vol1 = new File(dataDir, "data"+(2*1+1)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, false)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, false)); + DataNodeTestUtils.injectDataDirFailure(dn1Vol1, dn2Vol1); Path file1 = new Path("/test1"); DFSTestUtil.createFile(fs, file1, 1024, (short)2, 1L); @@ -323,14 +305,7 @@ public class TestDataNodeVolumeFailureReporting { // Make the first two volume directories on the first two datanodes // non-accessible. - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, - false)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol2, - false)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, - false)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol2, - false)); + DataNodeTestUtils.injectDataDirFailure(dn1Vol1, dn1Vol2, dn2Vol1, dn2Vol2); // Create file1 and wait for 3 replicas (ie all DNs can still store a block). // Then assert that all DNs are up, despite the volume failures. @@ -380,8 +355,8 @@ public class TestDataNodeVolumeFailureReporting { File dn1Vol2 = new File(dataDir, "data"+(2*0+2)); File dn2Vol1 = new File(dataDir, "data"+(2*1+1)); File dn2Vol2 = new File(dataDir, "data"+(2*1+2)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, false)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, false)); + DataNodeTestUtils.injectDataDirFailure(dn1Vol1); + DataNodeTestUtils.injectDataDirFailure(dn2Vol1); Path file1 = new Path("/test1"); DFSTestUtil.createFile(fs, file1, 1024, (short)2, 1L); @@ -449,8 +424,7 @@ public class TestDataNodeVolumeFailureReporting { // Replace failed volume with healthy volume and run reconfigure DataNode. // The failed volume information should be cleared. - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, true)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, true)); + DataNodeTestUtils.restoreDataDirFromFailure(dn1Vol1, dn2Vol1); reconfigureDataNode(dns.get(0), dn1Vol1, dn1Vol2); reconfigureDataNode(dns.get(1), dn2Vol1, dn2Vol2); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c238ae4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java index 73dc77c..5b7ac30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java @@ -76,10 +76,6 @@ public class TestDataNodeVolumeFailureToleration { @After public void tearDown() throws Exception { - for (int i = 0; i < 3; i++) { - FileUtil.setExecutable(new File(dataDir, "data"+(2*i+1)), true); - FileUtil.setExecutable(new File(dataDir, "data"+(2*i+2)), true); - } cluster.shutdown(); } @@ -152,7 +148,7 @@ public class TestDataNodeVolumeFailureToleration { // Fail a volume on the 2nd DN File dn2Vol1 = new File(dataDir, "data"+(2*1+1)); - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, false)); + DataNodeTestUtils.injectDataDirFailure(dn2Vol1); // Should only get two replicas (the first DN and the 3rd) Path file1 = new Path("/test1"); @@ -165,7 +161,7 @@ public class TestDataNodeVolumeFailureToleration { // If we restore the volume we should still only be able to get // two replicas since the DN is still considered dead. - assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, true)); + DataNodeTestUtils.restoreDataDirFromFailure(dn2Vol1); Path file2 = new Path("/test2"); DFSTestUtil.createFile(fs, file2, 1024, (short)3, 1L); DFSTestUtil.waitReplication(fs, file2, (short)2);
