Repository: hadoop Updated Branches: refs/heads/branch-2 076c68878 -> 9ab89a926
HDFS-8278. When computing max-size-to-move in Balancer, count only the storage with remaining >= default block size. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ab89a92 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ab89a92 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ab89a92 Branch: refs/heads/branch-2 Commit: 9ab89a9266acaa33007c888213d7b2cc2110eb60 Parents: 076c688 Author: Tsz-Wo Nicholas Sze <szets...@hortonworks.com> Authored: Mon Aug 17 17:55:25 2015 -0700 Committer: Tsz-Wo Nicholas Sze <szets...@hortonworks.com> Committed: Mon Aug 17 18:42:41 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/balancer/Balancer.java | 16 +++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ab89a92/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 31b197a..24ce942 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -458,6 +458,9 @@ Release 2.8.0 - UNRELEASED HDFS-8880. NameNode metrics logging. (Arpit Agarwal) + HDFS-8278. When computing max-size-to-move in Balancer, count only the + storage with remaining >= default block size. (szetszwo) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ab89a92/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 4f3f18e..16945df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -192,6 +192,7 @@ public class Balancer { private final boolean runDuringUpgrade; private final double threshold; private final long maxSizeToMove; + private final long defaultBlockSize; // all data node lists private final Collection<Source> overUtilized = new LinkedList<Source>(); @@ -270,6 +271,9 @@ public class Balancer { this.maxSizeToMove = getLong(conf, DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY, DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_DEFAULT); + this.defaultBlockSize = getLong(conf, + DFSConfigKeys.DFS_BLOCK_SIZE_KEY, + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); } private static long getCapacity(DatanodeStorageReport report, StorageType t) { @@ -282,11 +286,13 @@ public class Balancer { return capacity; } - private static long getRemaining(DatanodeStorageReport report, StorageType t) { + private long getRemaining(DatanodeStorageReport report, StorageType t) { long remaining = 0L; for(StorageReport r : report.getStorageReports()) { if (r.getStorage().getStorageType() == t) { - remaining += r.getRemaining(); + if (r.getRemaining() >= defaultBlockSize) { + remaining += r.getRemaining(); + } } } return remaining; @@ -323,7 +329,7 @@ public class Balancer { final double utilizationDiff = utilization - policy.getAvgUtilization(t); final double thresholdDiff = Math.abs(utilizationDiff) - threshold; final long maxSize2Move = computeMaxSize2Move(capacity, - getRemaining(r, t), utilizationDiff, threshold, maxSizeToMove); + getRemaining(r, t), utilizationDiff, maxSizeToMove); final StorageGroup g; if (utilizationDiff > 0) { @@ -360,8 +366,8 @@ public class Balancer { } private static long computeMaxSize2Move(final long capacity, final long remaining, - final double utilizationDiff, final double threshold, final long max) { - final double diff = Math.min(threshold, Math.abs(utilizationDiff)); + final double utilizationDiff, final long max) { + final double diff = Math.abs(utilizationDiff); long maxSizeToMove = percentage2bytes(diff, capacity); if (utilizationDiff < 0) { maxSizeToMove = Math.min(remaining, maxSizeToMove);
