HBASE-19913 Split TestStochasticLoadBalancer2
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/94dad844 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/94dad844 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/94dad844 Branch: refs/heads/HBASE-19064 Commit: 94dad844af239eafd09dac951bbc9ac2243fac82 Parents: 3622bb0 Author: zhangduo <zhang...@apache.org> Authored: Thu Feb 1 15:56:25 2018 +0800 Committer: zhangduo <zhang...@apache.org> Committed: Fri Feb 2 08:29:28 2018 +0800 ---------------------------------------------------------------------- .../master/balancer/BalancerTestBase2.java | 40 ++++++++ .../balancer/TestStochasticLoadBalancer2.java | 96 -------------------- ...TestStochasticLoadBalancerRegionReplica.java | 4 +- ...oadBalancerRegionReplicaHighReplication.java | 46 ++++++++++ ...icLoadBalancerRegionReplicaLargeCluster.java | 43 +++++++++ ...sticLoadBalancerRegionReplicaMidCluster.java | 43 +++++++++ ...onReplicaReplicationGreaterThanNumNodes.java | 46 ++++++++++ 7 files changed, 220 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase2.java new file mode 100644 index 0000000..dc952da --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase2.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.junit.After; +import org.junit.Before; + +public class BalancerTestBase2 extends BalancerTestBase { + + @Before + public void before() { + conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f); + conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L); + conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0); + conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec + conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.05f); + loadBalancer.setConf(conf); + } + + @After + public void after() { + // reset config to make sure balancer run + loadBalancer.setConf(conf); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java deleted file mode 100644 index da573da..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.balancer; - -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.FlakeyTests; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category({FlakeyTests.class, LargeTests.class}) -public class TestStochasticLoadBalancer2 extends BalancerTestBase { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestStochasticLoadBalancer2.class); - - @Before - public void before() { - conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f); - conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L); - conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0); - conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec - conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.05f); - loadBalancer.setConf(conf); - } - - @After - public void after() { - // reset config to make sure balancer run - loadBalancer.setConf(conf); - } - - @Test - public void testRegionReplicasOnMidCluster() { - int numNodes = 200; - int numRegions = 40 * 200; - int replication = 3; // 3 replicas per region - int numRegionsPerServer = 30; //all regions are mostly balanced - int numTables = 10; - testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true); - } - - @Test - public void testRegionReplicasOnLargeCluster() { - int numNodes = 1000; - int numRegions = 20 * numNodes; // 20 * replication regions per RS - int numRegionsPerServer = 19; // all servers except one - int numTables = 100; - int replication = 3; - testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true); - } - - @Test - public void testRegionReplicasOnMidClusterHighReplication() { - conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 4000000L); - conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec - loadBalancer.setConf(conf); - int numNodes = 80; - int numRegions = 6 * numNodes; - int replication = 80; // 80 replicas per region, one for each server - int numRegionsPerServer = 5; - int numTables = 10; - testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, true); - } - - @Test - public void testRegionReplicationOnMidClusterReplicationGreaterThanNumNodes() { - conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec - loadBalancer.setConf(conf); - int numNodes = 40; - int numRegions = 6 * 50; - int replication = 50; // 50 replicas per region, more than numNodes - int numRegionsPerServer = 6; - int numTables = 10; - testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, false); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java index cb48e1d..5b06f86 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java @@ -35,13 +35,13 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster; -import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -@Category({ MasterTests.class, LargeTests.class }) +@Category({ MasterTests.class, MediumTests.class }) public class TestStochasticLoadBalancerRegionReplica extends BalancerTestBase { @ClassRule http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaHighReplication.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaHighReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaHighReplication.java new file mode 100644 index 0000000..1253304 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaHighReplication.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, LargeTests.class }) +public class TestStochasticLoadBalancerRegionReplicaHighReplication extends BalancerTestBase2 { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestStochasticLoadBalancerRegionReplicaHighReplication.class); + + @Test + public void testRegionReplicasOnMidClusterHighReplication() { + conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 4000000L); + conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec + loadBalancer.setConf(conf); + int numNodes = 80; + int numRegions = 6 * numNodes; + int replication = 80; // 80 replicas per region, one for each server + int numRegionsPerServer = 5; + int numTables = 10; + testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, true); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaLargeCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaLargeCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaLargeCluster.java new file mode 100644 index 0000000..f43af33 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaLargeCluster.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, LargeTests.class }) +public class TestStochasticLoadBalancerRegionReplicaLargeCluster extends BalancerTestBase2 { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestStochasticLoadBalancerRegionReplicaLargeCluster.class); + + @Test + public void testRegionReplicasOnLargeCluster() { + int numNodes = 1000; + int numRegions = 20 * numNodes; // 20 * replication regions per RS + int numRegionsPerServer = 19; // all servers except one + int numTables = 100; + int replication = 3; + testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaMidCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaMidCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaMidCluster.java new file mode 100644 index 0000000..99c46a5 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaMidCluster.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, LargeTests.class }) +public class TestStochasticLoadBalancerRegionReplicaMidCluster extends BalancerTestBase2 { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestStochasticLoadBalancerRegionReplicaMidCluster.class); + + @Test + public void testRegionReplicasOnMidCluster() { + int numNodes = 200; + int numRegions = 40 * 200; + int replication = 3; // 3 replicas per region + int numRegionsPerServer = 30; // all regions are mostly balanced + int numTables = 10; + testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/94dad844/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes.java new file mode 100644 index 0000000..fd0cc98 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, LargeTests.class }) +public class TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes + extends BalancerTestBase2 { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule + .forClass(TestStochasticLoadBalancerRegionReplicaReplicationGreaterThanNumNodes.class); + + @Test + public void testRegionReplicationOnMidClusterReplicationGreaterThanNumNodes() { + conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec + loadBalancer.setConf(conf); + int numNodes = 40; + int numRegions = 6 * 50; + int replication = 50; // 50 replicas per region, more than numNodes + int numRegionsPerServer = 6; + int numTables = 10; + testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, false); + } +}
