Repository: hadoop Updated Branches: refs/heads/branch-2.7 47fcae7da -> b23b60e7b
HDFS-8824. Do not use small blocks for balancing the cluster. (cherry-picked from commit 61b9e5f7ff15daa0efd09e98efd70351f474c8cb) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b23b60e7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b23b60e7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b23b60e7 Branch: refs/heads/branch-2.7 Commit: b23b60e7b445f42a1ebed9b7f203566079007704 Parents: 47fcae7 Author: Tsz-Wo Nicholas Sze <[email protected]> Authored: Fri Aug 14 13:03:19 2015 -0700 Committer: Zhe Zhang <[email protected]> Committed: Fri Oct 7 11:01:29 2016 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +++ .../hadoop/hdfs/server/balancer/Balancer.java | 9 ++++- .../hadoop/hdfs/server/balancer/Dispatcher.java | 38 ++++++++++++-------- .../hdfs/server/balancer/TestBalancer.java | 10 ++++-- 5 files changed, 46 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b60e7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3350509..19e2bff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -50,6 +50,8 @@ Release 2.7.4 - UNRELEASED HDFS-10745. Directly resolve paths into INodesInPath. (Daryn Sharp via kihwal) + HDFS-8824. Do not use small blocks for balancing the cluster. (szetszwo) + OPTIMIZATIONS HDFS-10896. Move lock logging logic from FSNamesystem into FSNamesystemLock. http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b60e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 59df45c..e569fe9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -450,6 +450,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_BALANCER_ADDRESS_DEFAULT= "0.0.0.0:0"; public static final String DFS_BALANCER_KEYTAB_FILE_KEY = "dfs.balancer.keytab.file"; public static final String DFS_BALANCER_KERBEROS_PRINCIPAL_KEY = "dfs.balancer.kerberos.principal"; + public static final String DFS_BALANCER_GETBLOCKS_SIZE_KEY = "dfs.balancer.getBlocks.size"; + public static final long DFS_BALANCER_GETBLOCKS_SIZE_DEFAULT = 2L*1024*1024*1024; // 2GB + public static final String DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY = "dfs.balancer.getBlocks.min-block-size"; + public static final long DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_DEFAULT = 10L*1024*1024; // 10MB public static final String DFS_MOVER_MOVEDWINWIDTH_KEY = "dfs.mover.movedWinWidth"; public static final long DFS_MOVER_MOVEDWINWIDTH_DEFAULT = 5400*1000L; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b60e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 3a31ded..1772475 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -249,9 +249,16 @@ public class Balancer { DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY, DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT); + final long getBlocksSize = getLong(conf, + DFSConfigKeys.DFS_BALANCER_GETBLOCKS_SIZE_KEY, + DFSConfigKeys.DFS_BALANCER_GETBLOCKS_SIZE_DEFAULT); + final long getBlocksMinBlockSize = getLong(conf, + DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, + DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_DEFAULT); + this.dispatcher = new Dispatcher(theblockpool, p.nodesToBeIncluded, p.nodesToBeExcluded, movedWinWidth, moverThreads, dispatcherThreads, - maxConcurrentMovesPerNode, conf); + maxConcurrentMovesPerNode, getBlocksSize, getBlocksMinBlockSize, conf); this.threshold = p.threshold; this.policy = p.policy; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b60e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index e175330..c336cd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -85,9 +85,6 @@ import com.google.common.base.Preconditions; public class Dispatcher { static final Log LOG = LogFactory.getLog(Dispatcher.class); - private static final long GB = 1L << 30; // 1GB - private static final long MAX_BLOCKS_SIZE_TO_FETCH = 2 * GB; - private static final int MAX_NO_PENDING_MOVE_ITERATIONS = 5; /** * the period of time to delay the usage of a DataNode after hitting @@ -122,6 +119,9 @@ public class Dispatcher { /** The maximum number of concurrent blocks moves at a datanode */ private final int maxConcurrentMovesPerNode; + private final long getBlocksSize; + private final long getBlocksMinBlockSize; + static class Allocator { private final int max; private int count = 0; @@ -645,8 +645,9 @@ public class Dispatcher { * @return the total size of the received blocks in the number of bytes. */ private long getBlockList() throws IOException { - final long size = Math.min(MAX_BLOCKS_SIZE_TO_FETCH, blocksToReceive); + final long size = Math.min(getBlocksSize, blocksToReceive); final BlocksWithLocations newBlocks = nnc.getBlocks(getDatanodeInfo(), size); + if (LOG.isTraceEnabled()) { LOG.trace("getBlocks(" + getDatanodeInfo() + ", " + StringUtils.TraditionalBinaryPrefix.long2String(size, "B", 2) @@ -655,6 +656,11 @@ public class Dispatcher { long bytesReceived = 0; for (BlockWithLocations blk : newBlocks.getBlocks()) { + // Skip small blocks. + if (blk.getBlock().getNumBytes() < getBlocksMinBlockSize) { + continue; + } + bytesReceived += blk.getBlock().getNumBytes(); synchronized (globalBlocks) { final DBlock block = globalBlocks.get(blk.getBlock()); @@ -833,9 +839,19 @@ public class Dispatcher { } } + /** Constructor called by Mover. */ public Dispatcher(NameNodeConnector nnc, Set<String> includedNodes, Set<String> excludedNodes, long movedWinWidth, int moverThreads, int dispatcherThreads, int maxConcurrentMovesPerNode, Configuration conf) { + this(nnc, includedNodes, excludedNodes, movedWinWidth, + moverThreads, dispatcherThreads, maxConcurrentMovesPerNode, + 0L, 0L, conf); + } + + Dispatcher(NameNodeConnector nnc, Set<String> includedNodes, + Set<String> excludedNodes, long movedWinWidth, int moverThreads, + int dispatcherThreads, int maxConcurrentMovesPerNode, + long getBlocksSize, long getBlocksMinBlockSize, Configuration conf) { this.nnc = nnc; this.excludedNodes = excludedNodes; this.includedNodes = includedNodes; @@ -848,6 +864,9 @@ public class Dispatcher { this.moverThreadAllocator = new Allocator(moverThreads); this.maxConcurrentMovesPerNode = maxConcurrentMovesPerNode; + this.getBlocksSize = getBlocksSize; + this.getBlocksMinBlockSize = getBlocksMinBlockSize; + this.saslClient = new SaslDataTransferClient(conf, DataTransferSaslUtil.getSaslPropertiesResolver(conf), TrustedChannelResolver.getInstance(conf), nnc.fallbackToSimpleAuth); @@ -995,9 +1014,6 @@ public class Dispatcher { return getBytesMoved() - bytesLastMoved; } - /** The sleeping period before checking if block move is completed again */ - static private long blockMoveWaitTime = 30000L; - /** * Wait for all block move confirmations. * @return true if there is failed move execution @@ -1019,7 +1035,7 @@ public class Dispatcher { return hasFailure; // all pending queues are empty } try { - Thread.sleep(blockMoveWaitTime); + Thread.sleep(1000); } catch (InterruptedException ignored) { } } @@ -1134,12 +1150,6 @@ public class Dispatcher { movedBlocks.cleanup(); } - /** set the sleeping period for block move completion check */ - @VisibleForTesting - public static void setBlockMoveWaitTime(long time) { - blockMoveWaitTime = time; - } - @VisibleForTesting public static void setDelayAfterErrors(long time) { delayAfterErrors = time; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b60e7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 065e8a1..2ba5d55 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -205,8 +205,6 @@ public class TestBalancer { } public static void initTestSetup() { - Dispatcher.setBlockMoveWaitTime(1000L) ; - // do not create id file since it occupies the disk space NameNodeConnector.setWrite2IdFile(false); } @@ -215,9 +213,12 @@ public class TestBalancer { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500); conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); SimulatedFSDataset.setFactory(conf); + conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L); + conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L); } static void initConfWithRamDisk(Configuration conf) { @@ -227,6 +228,8 @@ public class TestBalancer { conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500); conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, 1); conf.setInt(DFS_DATANODE_RAM_DISK_LOW_WATERMARK_BYTES, DEFAULT_RAM_DISK_BLOCK_SIZE); + + conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L); } /* create a file with a length of <code>fileLen</code> */ @@ -1421,6 +1424,7 @@ public class TestBalancer { } /** + * Test special case. Two replicas belong to same block should not in same node. * We have 2 nodes. * We have a block in (DN0,SSD) and (DN1,DISK). @@ -1436,6 +1440,8 @@ public class TestBalancer { conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); + conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L); + int numOfDatanodes =2; final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(2) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
