Repository: hadoop Updated Branches: refs/heads/trunk 4ca3a6b21 -> 300f560fc
HDFS-14102. Performance improvement in BlockPlacementPolicyDefault. Contributed by Beluga Behr. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/300f560f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/300f560f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/300f560f Branch: refs/heads/trunk Commit: 300f560fcca879d212eadffaa59fbbb7017c9a3f Parents: 4ca3a6b Author: Giovanni Matteo Fumarola <gif...@apache.org> Authored: Wed Nov 28 11:33:22 2018 -0800 Committer: Giovanni Matteo Fumarola <gif...@apache.org> Committed: Wed Nov 28 11:33:22 2018 -0800 ---------------------------------------------------------------------- .../BlockPlacementPolicyDefault.java | 22 ++++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/300f560f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index d396845..5b8a8b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -71,6 +71,9 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { CHOOSE_RANDOM_REASONS = ThreadLocal .withInitial(() -> new HashMap<NodeNotChosenReason, Integer>()); + private static final BlockPlacementStatus ONE_RACK_PLACEMENT = + new BlockPlacementStatusDefault(1, 1, 1); + private enum NodeNotChosenReason { NOT_IN_SERVICE("the node is not in service"), NODE_STALE("the node is stale"), @@ -1029,22 +1032,23 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { @Override public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs, int numberOfReplicas) { - if (locs == null) + if (locs == null) { locs = DatanodeDescriptor.EMPTY_ARRAY; + } if (!clusterMap.hasClusterEverBeenMultiRack()) { // only one rack - return new BlockPlacementStatusDefault(1, 1, 1); + return ONE_RACK_PLACEMENT; } - int minRacks = 2; - minRacks = Math.min(minRacks, numberOfReplicas); + final int minRacks = Math.min(2, numberOfReplicas); // 1. Check that all locations are different. // 2. Count locations on different racks. - Set<String> racks = new TreeSet<>(); - for (DatanodeInfo dn : locs) - racks.add(dn.getNetworkLocation()); - return new BlockPlacementStatusDefault(racks.size(), minRacks, - clusterMap.getNumOfRacks()); + final long rackCount = Arrays.asList(locs).stream() + .map(dn -> dn.getNetworkLocation()).distinct().count(); + + return new BlockPlacementStatusDefault(Math.toIntExact(rackCount), + minRacks, clusterMap.getNumOfRacks()); } + /** * Decide whether deleting the specified replica of the block still makes * the block conform to the configured block placement policy. --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
