This is an automated email from the ASF dual-hosted git repository. arp pushed a commit to branch branch-2.8 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.8 by this push: new d71eda9 HDFS-13677. Dynamic refresh Disk configuration results in overwriting VolumeMap. Contributed by xuzq and Stephen O'Donnell. d71eda9 is described below commit d71eda92c6730f542be7ffd911cdc924d7e17b05 Author: Arpit Agarwal <a...@apache.org> AuthorDate: Mon May 6 13:34:38 2019 -0700 HDFS-13677. Dynamic refresh Disk configuration results in overwriting VolumeMap. Contributed by xuzq and Stephen O'Donnell. (cherry picked from commit 102c8fca10f3c626ab8bc47f818c8391a5c35289) (cherry picked from commit 4a1d51dea2149e8f458467467798e81b126b7cc5) --- .../datanode/fsdataset/impl/FsDatasetImpl.java | 2 +- .../server/datanode/fsdataset/impl/ReplicaMap.java | 14 ++++- .../datanode/TestDataNodeHotSwapVolumes.java | 71 ++++++++++++++++++++++ .../datanode/fsdataset/impl/TestReplicaMap.java | 22 +++++++ 4 files changed, 107 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index b14f9e9..d14033d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -415,7 +415,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> { LOG.error(errorMsg); throw new IOException(errorMsg); } - volumeMap.addAll(replicaMap); + volumeMap.mergeAll(replicaMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java index 5705792..e94670a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java @@ -126,7 +126,19 @@ class ReplicaMap { void addAll(ReplicaMap other) { map.putAll(other.map); } - + + + /** + * Merge all entries from the given replica map into the local replica map. + */ + void mergeAll(ReplicaMap other) { + for(String bp : other.getBlockPoolList()) { + for(ReplicaInfo r : other.map.get(bp)) { + add(bp, r); + } + } + } + /** * Remove the replica's meta information from the map that matches * the input block's id and generation stamp diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 08df71f..a07ecc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -47,6 +47,8 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.Test; @@ -393,6 +395,75 @@ public class TestDataNodeHotSwapVolumes { verifyFileLength(cluster.getFileSystem(), testFile, numBlocks); } + /** + * Test re-adding one volume with some blocks on a running MiniDFSCluster + * with only one NameNode to reproduce HDFS-13677. + */ + @Test(timeout=60000) + public void testReAddVolumeWithBlocks() + throws IOException, ReconfigurationException, + InterruptedException, TimeoutException { + startDFSCluster(1, 1); + String bpid = cluster.getNamesystem().getBlockPoolId(); + final int numBlocks = 10; + + Path testFile = new Path("/test"); + createFile(testFile, numBlocks); + + List<Map<DatanodeStorage, BlockListAsLongs>> blockReports = + cluster.getAllBlockReports(bpid); + assertEquals(1, blockReports.size()); // 1 DataNode + assertEquals(2, blockReports.get(0).size()); // 2 volumes + + // Now remove the second volume + DataNode dn = cluster.getDataNodes().get(0); + Collection<String> oldDirs = getDataDirs(dn); + String newDirs = oldDirs.iterator().next(); // Keep the first volume. + assertThat( + "DN did not update its own config", + dn.reconfigurePropertyImpl( + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs), + is(dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY))); + assertFileLocksReleased( + new ArrayList<String>(oldDirs).subList(1, oldDirs.size())); + + // Now create another file - the first volume should have 15 blocks + // and 5 blocks on the previously removed volume + createFile(new Path("/test2"), numBlocks); + dn.scheduleAllBlockReport(0); + blockReports = cluster.getAllBlockReports(bpid); + + assertEquals(1, blockReports.size()); // 1 DataNode + assertEquals(1, blockReports.get(0).size()); // 1 volume + for (BlockListAsLongs blockList : blockReports.get(0).values()) { + assertEquals(15, blockList.getNumberOfBlocks()); + } + + // Now add the original volume back again and ensure 15 blocks are reported + assertThat( + "DN did not update its own config", + dn.reconfigurePropertyImpl( + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, + StringUtils.join(",", oldDirs)), + is(dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY))); + dn.scheduleAllBlockReport(0); + blockReports = cluster.getAllBlockReports(bpid); + + assertEquals(1, blockReports.size()); // 1 DataNode + assertEquals(2, blockReports.get(0).size()); // 2 volumes + + // The order of the block reports is not guaranteed. As we expect 2, get the + // max block count and the min block count and then assert on that. + int minNumBlocks = Integer.MAX_VALUE; + int maxNumBlocks = Integer.MIN_VALUE; + for (BlockListAsLongs blockList : blockReports.get(0).values()) { + minNumBlocks = Math.min(minNumBlocks, blockList.getNumberOfBlocks()); + maxNumBlocks = Math.max(maxNumBlocks, blockList.getNumberOfBlocks()); + } + assertEquals(5, minNumBlocks); + assertEquals(15, maxNumBlocks); + } + @Test(timeout=60000) public void testAddVolumesDuringWrite() throws IOException, InterruptedException, TimeoutException, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaMap.java index 4fa91b0..1059c08 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaMap.java @@ -108,4 +108,26 @@ public class TestReplicaMap { map.add(bpid, new FinalizedReplica(block, null, null)); assertNotNull(map.remove(bpid, block.getBlockId())); } + + @Test + public void testMergeAll() { + ReplicaMap temReplicaMap = new ReplicaMap(new AutoCloseableLock()); + Block tmpBlock = new Block(5678, 5678, 5678); + temReplicaMap.add(bpid, new FinalizedReplica(tmpBlock, null, null)); + + map.mergeAll(temReplicaMap); + assertNotNull(map.get(bpid, 1234)); + assertNotNull(map.get(bpid, 5678)); + } + + @Test + public void testAddAll() { + ReplicaMap temReplicaMap = new ReplicaMap(new AutoCloseableLock()); + Block tmpBlock = new Block(5678, 5678, 5678); + temReplicaMap.add(bpid, new FinalizedReplica(tmpBlock, null, null)); + + map.addAll(temReplicaMap); + assertNull(map.get(bpid, 1234)); + assertNotNull(map.get(bpid, 5678)); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org