Repository: hadoop Updated Branches: refs/heads/HDFS-7240 c78518749 -> 8a070ee48
HDFS-8278. When computing max-size-to-move in Balancer, count only the storage with remaining >= default block size. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51a00964 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51a00964 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51a00964 Branch: refs/heads/HDFS-7240 Commit: 51a00964da0e399718d1cec25ff692a32d7642b7 Parents: eee4d71 Author: Tsz-Wo Nicholas Sze <szets...@hortonworks.com> Authored: Mon Aug 17 17:55:25 2015 -0700 Committer: Tsz-Wo Nicholas Sze <szets...@hortonworks.com> Committed: Mon Aug 17 17:55:25 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/balancer/Balancer.java | 16 +++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/51a00964/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7d05bc2..d1b04dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -803,6 +803,9 @@ Release 2.8.0 - UNRELEASED HDFS-8880. NameNode metrics logging. (Arpit Agarwal) + HDFS-8278. When computing max-size-to-move in Balancer, count only the + storage with remaining >= default block size. (szetszwo) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/51a00964/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 6fc024e..e92ed81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -191,6 +191,7 @@ public class Balancer { private final boolean runDuringUpgrade; private final double threshold; private final long maxSizeToMove; + private final long defaultBlockSize; // all data node lists private final Collection<Source> overUtilized = new LinkedList<Source>(); @@ -269,6 +270,9 @@ public class Balancer { this.maxSizeToMove = getLong(conf, DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY, DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_DEFAULT); + this.defaultBlockSize = getLong(conf, + DFSConfigKeys.DFS_BLOCK_SIZE_KEY, + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); } private static long getCapacity(DatanodeStorageReport report, StorageType t) { @@ -281,11 +285,13 @@ public class Balancer { return capacity; } - private static long getRemaining(DatanodeStorageReport report, StorageType t) { + private long getRemaining(DatanodeStorageReport report, StorageType t) { long remaining = 0L; for(StorageReport r : report.getStorageReports()) { if (r.getStorage().getStorageType() == t) { - remaining += r.getRemaining(); + if (r.getRemaining() >= defaultBlockSize) { + remaining += r.getRemaining(); + } } } return remaining; @@ -322,7 +328,7 @@ public class Balancer { final double utilizationDiff = utilization - policy.getAvgUtilization(t); final double thresholdDiff = Math.abs(utilizationDiff) - threshold; final long maxSize2Move = computeMaxSize2Move(capacity, - getRemaining(r, t), utilizationDiff, threshold, maxSizeToMove); + getRemaining(r, t), utilizationDiff, maxSizeToMove); final StorageGroup g; if (utilizationDiff > 0) { @@ -359,8 +365,8 @@ public class Balancer { } private static long computeMaxSize2Move(final long capacity, final long remaining, - final double utilizationDiff, final double threshold, final long max) { - final double diff = Math.min(threshold, Math.abs(utilizationDiff)); + final double utilizationDiff, final long max) { + final double diff = Math.abs(utilizationDiff); long maxSizeToMove = percentage2bytes(diff, capacity); if (utilizationDiff < 0) { maxSizeToMove = Math.min(remaining, maxSizeToMove);
