Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 30dc4334b -> 089169e4e
HDFS-9313. Possible NullPointerException in BlockManager if no excess replica
can be chosen. Contributed by Ming Ma. Branch-2.7 patch contributed by Xiao
Chen.
Change-Id: Iadaee6be2b44f7b0322d7b0e58f8fc983ef463f6
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/089169e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/089169e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/089169e4
Branch: refs/heads/branch-2.6
Commit: 089169e4e571c16ebce0ce59801833bca3d5c7c3
Parents: 30dc433
Author: Zhe Zhang <[email protected]>
Authored: Thu Dec 3 23:19:17 2015 -0800
Committer: Zhe Zhang <[email protected]>
Committed: Thu Dec 3 23:24:21 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../BlockPlacementPolicyDefault.java | 5 ++++
.../blockmanagement/TestReplicationPolicy.java | 31 ++++++++++++++++++++
3 files changed, 38 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/089169e4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5c987d2..3ac0be6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -11,6 +11,8 @@ Release 2.6.4 - UNRELEASED
HDFS-8647. Abstract BlockManager's rack policy into BlockPlacementPolicy.
(Brahma Reddy Battula via mingma)
+ HDFS-9313. Possible NullPointerException in BlockManager if no excess
+ replica can be chosen. (mingma)
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/089169e4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index bde846e..136d03e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -971,6 +971,11 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
excessTypes);
}
firstOne = false;
+ if (cur == null) {
+ LOG.warn("No excess replica can be found. excessTypes: "+excessTypes+
+ ". moreThanOne: "+moreThanOne+". exactlyOne: "+exactlyOne+".");
+ break;
+ }
// adjust rackmap, moreThanOne, and exactlyOne
adjustSetsWithChosenReplica(rackMap, moreThanOne, exactlyOne, cur);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/089169e4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 3484777..b392d27 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -1058,6 +1058,14 @@ public class TestReplicationPolicy {
BlockStoragePolicySuite POLICY_SUITE = BlockStoragePolicySuite
.createDefaultSuite();
BlockStoragePolicy storagePolicy = POLICY_SUITE.getDefaultPolicy();
+ DatanodeStorageInfo excessSSD = DFSTestUtil.createDatanodeStorageInfo(
+ "Storage-excess-SSD-ID", "localhost",
+ storages[0].getDatanodeDescriptor().getNetworkLocation(),
+ "foo.com", StorageType.SSD);
+ updateHeartbeatWithUsage(excessSSD.getDatanodeDescriptor(),
+ 2* HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
+ 2* HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0,
+ 0);
// use delete hint case.
@@ -1080,6 +1088,29 @@ public class TestReplicationPolicy {
excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
excessTypes, storages[3].getDatanodeDescriptor(), null);
assertTrue(excessReplicas.contains(excessStorage));
+
+
+ // The block was initially created on excessSSD(rack r1),
+ // storages[4](rack r3) and storages[5](rack r3) with
+ // ONESSD_STORAGE_POLICY_NAME storage policy.
+ // Right after balancer moves the block from storages[5] to
+ // storages[3](rack r2), the application changes the storage policy from
+ // ONESSD_STORAGE_POLICY_NAME to HOT_STORAGE_POLICY_ID. In this case,
+ // no replica can be chosen as the excessive replica as
+ // chooseReplicasToDelete only considers storages[4] and storages[5] that
+ // are the same rack. But neither's storage type is SSD.
+ // TODO BlockPlacementPolicyDefault should be able to delete excessSSD.
+ nonExcess.clear();
+ nonExcess.add(excessSSD);
+ nonExcess.add(storages[3]);
+ nonExcess.add(storages[4]);
+ nonExcess.add(storages[5]);
+ excessTypes = storagePolicy.chooseExcess((short) 3,
+ DatanodeStorageInfo.toStorageTypes(nonExcess));
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessTypes, storages[3].getDatanodeDescriptor(),
+ storages[5].getDatanodeDescriptor());
+ assertTrue(excessReplicas.size() == 0);
}
@Test
