HDFS-8666. Speedup the TestMover test. Contributed by Walter Su.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/152e5df3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/152e5df3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/152e5df3 Branch: refs/heads/HADOOP-12111 Commit: 152e5df3b65394c2939d6de3c4a649a207bb58d3 Parents: b5cdf78 Author: Jing Zhao <[email protected]> Authored: Wed Jul 1 14:59:50 2015 -0700 Committer: Jing Zhao <[email protected]> Committed: Wed Jul 1 14:59:50 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/server/mover/TestMover.java | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/152e5df3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3736932..ec37542 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -694,6 +694,8 @@ Release 2.8.0 - UNRELEASED HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison via Colin P. McCabe) + HDFS-8666. Speedup the TestMover tests. (Walter Su via jing9) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/152e5df3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java index 49e2b23..899b5c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.balancer.Dispatcher.DBlock; import org.apache.hadoop.hdfs.server.balancer.ExitStatus; import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector; +import org.apache.hadoop.hdfs.server.balancer.TestBalancer; import org.apache.hadoop.hdfs.server.mover.Mover.MLocation; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.test.GenericTestUtils; @@ -46,6 +47,21 @@ import org.junit.Assert; import org.junit.Test; public class TestMover { + + static final int DEFAULT_BLOCK_SIZE = 100; + + static { + TestBalancer.initTestSetup(); + } + + static void initConf(Configuration conf) { + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); + conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE); + conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); + conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L); + } + static Mover newMover(Configuration conf) throws IOException { final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf); Assert.assertEquals(1, namenodes.size()); @@ -97,6 +113,7 @@ public class TestMover { @Test public void testScheduleBlockWithinSameNode() throws Exception { final Configuration conf = new HdfsConfiguration(); + initConf(conf); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(3) .storageTypes( @@ -285,6 +302,7 @@ public class TestMover { public void testTwoReplicaSameStorageTypeShouldNotSelect() throws Exception { // HDFS-8147 final Configuration conf = new HdfsConfiguration(); + initConf(conf); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(3) .storageTypes( @@ -361,6 +379,7 @@ public class TestMover { public void testMoverFailedRetry() throws Exception { // HDFS-8147 final Configuration conf = new HdfsConfiguration(); + initConf(conf); conf.set(DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, "2"); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(3)
