Repository: hadoop Updated Branches: refs/heads/branch-2 c74e42b4a -> 5a3db2156
HDFS-8056. Decommissioned dead nodes should continue to be counted as dead after NN restart. (mingma) (cherry picked from commit 1c4951a7a09433fbbcfe26f243d6c2d8043c71bb) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a3db215 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a3db215 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a3db215 Branch: refs/heads/branch-2 Commit: 5a3db21563d1f7303d263efcf5631c447bda4307 Parents: c74e42b Author: Ming Ma <[email protected]> Authored: Thu Nov 19 10:04:01 2015 -0800 Committer: Ming Ma <[email protected]> Committed: Thu Nov 19 10:04:40 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/blockmanagement/DatanodeManager.java | 5 ++- .../apache/hadoop/hdfs/TestDecommission.java | 35 ++++++++++++++++++++ .../blockmanagement/TestHostFileManager.java | 2 +- 4 files changed, 43 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3db215/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dcf0ecf..54fefb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -798,6 +798,9 @@ Release 2.8.0 - UNRELEASED HDFS-9252. Change TestFileTruncate to use FsDatasetTestUtils to get block file size and genstamp. (Lei (Eddy) Xu via cmccabe) + HDFS-8056. Decommissioned dead nodes should continue to be counted as dead + after NN restart. (mingma) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3db215/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 870e5ad..49f4100 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1267,7 +1267,7 @@ public class DatanodeManager { if (listDeadNodes) { for (InetSocketAddress addr : includedNodes) { - if (foundNodes.matchedBy(addr) || excludedNodes.match(addr)) { + if (foundNodes.matchedBy(addr)) { continue; } // The remaining nodes are ones that are referenced by the hosts @@ -1284,6 +1284,9 @@ public class DatanodeManager { addr.getPort() == 0 ? defaultXferPort : addr.getPort(), defaultInfoPort, defaultInfoSecurePort, defaultIpcPort)); setDatanodeDead(dn); + if (excludedNodes.match(addr)) { + dn.setDecommissioned(); + } nodes.add(dn); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3db215/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java index 0dce5d3..396e50c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java @@ -925,6 +925,41 @@ public class TestDecommission { } /** + * Tests dead node count after restart of namenode + **/ + @Test(timeout=360000) + public void testDeadNodeCountAfterNamenodeRestart()throws Exception { + LOG.info("Starting test testDeadNodeCountAfterNamenodeRestart"); + int numNamenodes = 1; + int numDatanodes = 2; + + startCluster(numNamenodes, numDatanodes, conf); + + DFSClient client = getDfsClient(cluster.getNameNode(), conf); + DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); + DatanodeInfo excludedDatanode = info[0]; + String excludedDatanodeName = info[0].getXferAddr(); + + writeConfigFile(hostsFile, new ArrayList<String>(Arrays.asList( + excludedDatanodeName, info[1].getXferAddr()))); + decommissionNode(0, excludedDatanode.getDatanodeUuid(), null, + AdminStates.DECOMMISSIONED); + + cluster.stopDataNode(excludedDatanodeName); + DFSTestUtil.waitForDatanodeState( + cluster, excludedDatanode.getDatanodeUuid(), false, 20000); + + //Restart the namenode + cluster.restartNameNode(); + + assertEquals("There should be one node alive", 1, + client.datanodeReport(DatanodeReportType.LIVE).length); + assertEquals("There should be one node dead", 1, + client.datanodeReport(DatanodeReportType.DEAD).length); + cluster.shutdown(); + } + + /** * Test using a "registration name" in a host include file. * * Registration names are DataNode names specified in the configuration by http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3db215/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java index c65b580..6f17040 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java @@ -151,7 +151,7 @@ public class TestHostFileManager { Assert.assertEquals(1, dm.getDatanodeListForReport(HdfsConstants .DatanodeReportType.DEAD).size()); excludedNodes.add(entry("127.0.0.3")); - Assert.assertEquals(0, dm.getDatanodeListForReport(HdfsConstants + Assert.assertEquals(1, dm.getDatanodeListForReport(HdfsConstants .DatanodeReportType.DEAD).size()); } }
