Repository: hadoop Updated Branches: refs/heads/HDFS-9806 52f562486 -> 37acbc85f
HDFS-12894. [READ] Skip setting block count of ProvidedDatanodeStorageInfo on DN registration update Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37acbc85 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37acbc85 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37acbc85 Branch: refs/heads/HDFS-9806 Commit: 37acbc85fce9e1ade9ededc2175710a2d58dbc0a Parents: 52f5624 Author: Virajith Jalaparti <viraj...@apache.org> Authored: Tue Dec 5 17:55:32 2017 -0800 Committer: Virajith Jalaparti <viraj...@apache.org> Committed: Tue Dec 5 17:55:32 2017 -0800 ---------------------------------------------------------------------- .../server/blockmanagement/BlockManager.java | 5 +++++ .../blockmanagement/DatanodeDescriptor.java | 4 +++- .../TestNameNodeProvidedImplementation.java | 20 +++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/37acbc85/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 3c2822d..7ca1a1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -4904,4 +4904,9 @@ public class BlockManager implements BlockStatsMXBean { } return i; } + + @VisibleForTesting + public ProvidedStorageMap getProvidedStorageMap() { + return providedStorageMap; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/37acbc85/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 83c608f..fc58708 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -919,7 +919,9 @@ public class DatanodeDescriptor extends DatanodeInfo { // must re-process IBR after re-registration for(DatanodeStorageInfo storage : getStorageInfos()) { - storage.setBlockReportCount(0); + if (storage.getStorageType() != StorageType.PROVIDED) { + storage.setBlockReportCount(0); + } } heartbeatedSinceRegistration = false; forceRegistration = false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/37acbc85/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java index deaf9d5..d057247 100644 --- a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java +++ b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java @@ -559,7 +559,9 @@ public class TestNameNodeProvidedImplementation { DataNode providedDatanode2 = cluster.getDataNodes().get(1); DFSClient client = new DFSClient(new InetSocketAddress("localhost", - cluster.getNameNodePort()), cluster.getConfiguration(0)); + cluster.getNameNodePort()), cluster.getConfiguration(0)); + + DatanodeStorageInfo providedDNInfo = getProvidedDatanodeStorageInfo(); if (numFiles >= 1) { String filename = "/" + filePrefix + (numFiles - 1) + fileSuffix; @@ -596,10 +598,15 @@ public class TestNameNodeProvidedImplementation { providedDatanode2.getDatanodeId().getXferAddr()); getAndCheckBlockLocations(client, filename, baseFileLen, 1, 0); + // BR count for the provided ProvidedDatanodeStorageInfo should reset to + // 0, when all DNs with PROVIDED storage fail. + assertEquals(0, providedDNInfo.getBlockReportCount()); //restart the provided datanode cluster.restartDataNode(providedDNProperties1, true); cluster.waitActive(); + assertEquals(1, providedDNInfo.getBlockReportCount()); + //should find the block on the 1st provided datanode now dnInfos = getAndCheckBlockLocations(client, filename, baseFileLen, 1, 1); //not comparing UUIDs as the datanode can now have a different one. @@ -621,6 +628,8 @@ public class TestNameNodeProvidedImplementation { false); DataNode providedDatanode = cluster.getDataNodes().get(0); + DatanodeStorageInfo providedDNInfo = getProvidedDatanodeStorageInfo(); + int initialBRCount = providedDNInfo.getBlockReportCount(); for (int i= 0; i < numFiles; i++) { // expect to have 2 locations as we have 2 provided Datanodes. verifyFileLocation(i, 2); @@ -631,10 +640,19 @@ public class TestNameNodeProvidedImplementation { cluster.waitActive(); cluster.triggerHeartbeats(); Thread.sleep(1000); + // the report count should just continue to increase. + assertEquals(initialBRCount + i + 1, + providedDNInfo.getBlockReportCount()); verifyFileLocation(i, 2); } } + private DatanodeStorageInfo getProvidedDatanodeStorageInfo() { + ProvidedStorageMap providedStorageMap = + cluster.getNamesystem().getBlockManager().getProvidedStorageMap(); + return providedStorageMap.getProvidedStorageInfo(); + } + @Test(timeout=30000) public void testNamenodeRestart() throws Exception { createImage(new FSTreeWalk(NAMEPATH, conf), NNDIRPATH, --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org