Repository: hadoop Updated Branches: refs/heads/branch-2.7 6a30d93aa -> b39de3b0b
HDFS-9313. Possible NullPointerException in BlockManager if no excess replica can be chosen. Contributed by Ming Ma. Branch-2.7 patch contributed by Xiao Chen. Change-Id: Iadaee6be2b44f7b0322d7b0e58f8fc983ef463f6 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b39de3b0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b39de3b0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b39de3b0 Branch: refs/heads/branch-2.7 Commit: b39de3b0bd5e96b4279b7ea7799220bfb2c61e75 Parents: 6a30d93 Author: Zhe Zhang <[email protected]> Authored: Thu Dec 3 23:19:17 2015 -0800 Committer: Zhe Zhang <[email protected]> Committed: Thu Dec 3 23:19:17 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../BlockPlacementPolicyDefault.java | 5 ++++ .../blockmanagement/TestReplicationPolicy.java | 31 ++++++++++++++++++++ 3 files changed, 39 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39de3b0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ea831c4..bcc24ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -27,6 +27,9 @@ Release 2.7.3 - UNRELEASED HDFS-6481. DatanodeManager#getDatanodeStorageInfos() should check the length of storageIDs. (szetszwo via Arpit Agarwal) + HDFS-9313. Possible NullPointerException in BlockManager if no excess + replica can be chosen. (mingma) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39de3b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index ecbbcbb..cbf5cfc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -979,6 +979,11 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { excessTypes); } firstOne = false; + if (cur == null) { + LOG.warn("No excess replica can be found. excessTypes: "+excessTypes+ + ". moreThanOne: "+moreThanOne+". exactlyOne: "+exactlyOne+"."); + break; + } // adjust rackmap, moreThanOne, and exactlyOne adjustSetsWithChosenReplica(rackMap, moreThanOne, exactlyOne, cur); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39de3b0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index 0ab6739..3d96f20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -1072,6 +1072,14 @@ public class TestReplicationPolicy { BlockStoragePolicySuite POLICY_SUITE = BlockStoragePolicySuite .createDefaultSuite(); BlockStoragePolicy storagePolicy = POLICY_SUITE.getDefaultPolicy(); + DatanodeStorageInfo excessSSD = DFSTestUtil.createDatanodeStorageInfo( + "Storage-excess-SSD-ID", "localhost", + storages[0].getDatanodeDescriptor().getNetworkLocation(), + "foo.com", StorageType.SSD); + updateHeartbeatWithUsage(excessSSD.getDatanodeDescriptor(), + 2* HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, + 2* HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, + 0); // use delete hint case. @@ -1094,6 +1102,29 @@ public class TestReplicationPolicy { excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3, excessTypes, storages[3].getDatanodeDescriptor(), null); assertTrue(excessReplicas.contains(excessStorage)); + + + // The block was initially created on excessSSD(rack r1), + // storages[4](rack r3) and storages[5](rack r3) with + // ONESSD_STORAGE_POLICY_NAME storage policy. + // Right after balancer moves the block from storages[5] to + // storages[3](rack r2), the application changes the storage policy from + // ONESSD_STORAGE_POLICY_NAME to HOT_STORAGE_POLICY_ID. In this case, + // no replica can be chosen as the excessive replica as + // chooseReplicasToDelete only considers storages[4] and storages[5] that + // are the same rack. But neither's storage type is SSD. + // TODO BlockPlacementPolicyDefault should be able to delete excessSSD. + nonExcess.clear(); + nonExcess.add(excessSSD); + nonExcess.add(storages[3]); + nonExcess.add(storages[4]); + nonExcess.add(storages[5]); + excessTypes = storagePolicy.chooseExcess((short) 3, + DatanodeStorageInfo.toStorageTypes(nonExcess)); + excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3, + excessTypes, storages[3].getDatanodeDescriptor(), + storages[5].getDatanodeDescriptor()); + assertTrue(excessReplicas.size() == 0); } @Test
