Repository: hbase Updated Branches: refs/heads/master a36ffdaff -> 683f3b3d5
http://git-wip-us.apache.org/repos/asf/hbase/blob/683f3b3d/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java index 6a9f97d..a074655 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.master.balancer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -37,18 +36,16 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.RegionReplicaUtil; -import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.MoveRegionAction; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -270,7 +267,7 @@ public class TestBaseLoadBalancer extends BalancerTestBase { // cluster is created (constructor code) would make sure the indices of // the servers are in the order in which it is inserted in the clusterState // map (linkedhashmap is important). A similar thing applies to the region lists - Cluster cluster = new Cluster(master, clusterState, null, null, null, null, rackManager); + Cluster cluster = new Cluster(master, clusterState, null, null, null, rackManager); // check whether a move of region1 from servers[0] to servers[1] would lower // the availability of region1 assertTrue(cluster.wouldLowerAvailability(hri1, servers[1])); @@ -287,7 +284,7 @@ public class TestBaseLoadBalancer extends BalancerTestBase { // now lets have servers[1] host replica_of_region2 list1.add(RegionReplicaUtil.getRegionInfoForReplica(hri3, 1)); // create a new clusterState with the above change - cluster = new Cluster(master, clusterState, null, null, null, null, rackManager); + cluster = new Cluster(master, clusterState, null, null, null, rackManager); // now check whether a move of a replica from servers[0] to servers[1] would lower // the availability of region2 assertTrue(cluster.wouldLowerAvailability(hri3, servers[1])); @@ -299,14 +296,14 @@ public class TestBaseLoadBalancer extends BalancerTestBase { clusterState.put(servers[6], list2); //servers[6], rack2 hosts region2 clusterState.put(servers[10], new ArrayList<HRegionInfo>()); //servers[10], rack3 hosts no region // create a cluster with the above clusterState - cluster = new Cluster(master, clusterState, null, null, null, null, rackManager); + cluster = new Cluster(master, clusterState, null, null, null, rackManager); // check whether a move of region1 from servers[0],rack1 to servers[6],rack2 would // lower the availability assertTrue(cluster.wouldLowerAvailability(hri1, servers[0])); // now create a cluster without the rack manager - cluster = new Cluster(master, clusterState, null, null, null, null, null); + cluster = new Cluster(master, clusterState, null, null, null, null); // now repeat check whether a move of region1 from servers[0] to servers[6] would // lower the availability assertTrue(!cluster.wouldLowerAvailability(hri1, servers[6])); @@ -339,7 +336,7 @@ public class TestBaseLoadBalancer extends BalancerTestBase { // cluster is created (constructor code) would make sure the indices of // the servers are in the order in which it is inserted in the clusterState // map (linkedhashmap is important). - Cluster cluster = new Cluster(master, clusterState, null, null, null, null, rackManager); + Cluster cluster = new Cluster(master, clusterState, null, null, null, rackManager); // check whether moving region1 from servers[1] to servers[2] would lower availability assertTrue(!cluster.wouldLowerAvailability(hri1, servers[2])); @@ -359,7 +356,7 @@ public class TestBaseLoadBalancer extends BalancerTestBase { clusterState.put(servers[6], list2); //servers[6], rack2 hosts region2 clusterState.put(servers[12], list3); //servers[12], rack3 hosts replica_of_region2 // create a cluster with the above clusterState - cluster = new Cluster(master, clusterState, null, null, null, null, rackManager); + cluster = new Cluster(master, clusterState, null, null, null, rackManager); // check whether a move of replica_of_region2 from servers[12],rack3 to servers[0],rack1 would // lower the availability assertTrue(!cluster.wouldLowerAvailability(hri4, servers[0])); @@ -445,7 +442,7 @@ public class TestBaseLoadBalancer extends BalancerTestBase { assignRegions(regions, oldServers, clusterState); // should not throw exception: - BaseLoadBalancer.Cluster cluster = new Cluster(null, clusterState, null, null, null, null, null); + BaseLoadBalancer.Cluster cluster = new Cluster(null, clusterState, null, null, null, null); assertEquals(101 + 9, cluster.numRegions); assertEquals(10, cluster.numServers); // only 10 servers because they share the same host + port } @@ -487,7 +484,7 @@ public class TestBaseLoadBalancer extends BalancerTestBase { when(locationFinder.getTopBlockLocations(regions.get(43))).thenReturn( Lists.newArrayList(ServerName.valueOf("foo", 0, 0))); // this server does not exists in clusterStatus - BaseLoadBalancer.Cluster cluster = new Cluster(null, clusterState, null, locationFinder, null, null, null); + BaseLoadBalancer.Cluster cluster = new Cluster(null, clusterState, null, locationFinder, null, null); int r0 = ArrayUtils.indexOf(cluster.regions, regions.get(0)); // this is ok, it is just a test int r1 = ArrayUtils.indexOf(cluster.regions, regions.get(1)); @@ -523,49 +520,4 @@ public class TestBaseLoadBalancer extends BalancerTestBase { assertEquals(1, cluster.regionLocations[r43].length); assertEquals(-1, cluster.regionLocations[r43][0]); } - - @Test - public void testBackupMastersExcluded() throws HBaseIOException { - ClusterStatus st = Mockito.mock(ClusterStatus.class); - ArrayList<ServerName> backupMasters = new ArrayList<ServerName>(); - ServerName backupMaster = ServerName.valueOf("fake-backupmaster", 0, 1L); - backupMasters.add(backupMaster); - BaseLoadBalancer balancer = (BaseLoadBalancer)loadBalancer; - balancer.usingBackupMasters = false; - Mockito.when(st.getBackupMasters()).thenReturn(backupMasters); - loadBalancer.setClusterStatus(st); - assertEquals(1, balancer.excludedServers.size()); - assertTrue(balancer.excludedServers.contains(backupMaster)); - - // Round robin assignment - List<HRegionInfo> regions = randomRegions(1); - HRegionInfo region = regions.get(0); - assertNull(loadBalancer.randomAssignment(region, backupMasters)); - assertNull(loadBalancer.roundRobinAssignment(regions, backupMasters)); - HashMap<HRegionInfo, ServerName> assignments = new HashMap<HRegionInfo, ServerName>(); - assignments.put(region, backupMaster); - assertNull(loadBalancer.retainAssignment(assignments, backupMasters)); - ArrayList<ServerName> servers = new ArrayList<ServerName>(backupMasters); - ServerName sn = ServerName.valueOf("fake-rs", 0, 1L); - servers.add(sn); - assertEquals(sn, loadBalancer.randomAssignment(region, servers)); - Map<ServerName, List<HRegionInfo>> plans = - loadBalancer.roundRobinAssignment(regions, servers); - assertEquals(1, plans.size()); - assertTrue(plans.get(sn).contains(region)); - - // Retain assignment - plans = loadBalancer.retainAssignment(assignments, servers); - assertEquals(1, plans.size()); - assertTrue(plans.get(sn).contains(region)); - - // Filter backup masters for balance cluster - Map<ServerName, List<HRegionInfo>> clusterMap = - new HashMap<ServerName, List<HRegionInfo>>(); - clusterMap.put(backupMaster, new ArrayList<HRegionInfo>()); - clusterMap.put(sn, new ArrayList<HRegionInfo>()); - balancer.filterExcludedServers(clusterMap); - assertTrue(clusterMap.containsKey(sn)); - assertEquals(1, clusterMap.size()); - } } http://git-wip-us.apache.org/repos/asf/hbase/blob/683f3b3d/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDefaultLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDefaultLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDefaultLoadBalancer.java index 15ff805..d905c26 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDefaultLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDefaultLoadBalancer.java @@ -17,14 +17,8 @@ */ package org.apache.hadoop.hbase.master.balancer; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.TreeMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,7 +26,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -40,7 +33,6 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.mockito.Mockito; /** * Test the load balancer that is created by default. @@ -134,43 +126,4 @@ public class TestDefaultLoadBalancer extends BalancerTestBase { } } } - - @Test - public void testBalancerClusterWithBackupMaster() throws Exception { - SimpleLoadBalancer balancer = Mockito.spy(new SimpleLoadBalancer()); - balancer.setConf(HBaseConfiguration.create()); - List<ServerName> backupMasters = new ArrayList<ServerName>(); - ServerName backupMaster = ServerName.parseServerName("backup:1:1"); - ServerName rs = ServerName.parseServerName("rs:1:1"); - backupMasters.add(backupMaster); - Mockito.doReturn(backupMasters).when(balancer).getBackupMasters(); - Map<ServerName, List<HRegionInfo>> servers = new TreeMap<ServerName, List<HRegionInfo>>(); - List<HRegionInfo> regions = new ArrayList<HRegionInfo>(); - TableName table = TableName.valueOf("test"); - regions.add(new HRegionInfo(table)); - servers.put(backupMaster, regions); - regions = new ArrayList<HRegionInfo>(); - balancer.backupMasterWeight = 4; - for (int i=0; i<4; i++) { - regions.add(new HRegionInfo(table)); - } - servers.put(rs, regions); - List<RegionPlan> plans = balancer.balanceCluster(servers); - assertNull(plans); - - // Reset the cluster map - regions = new ArrayList<HRegionInfo>(); - for (int i=0; i<2; i++) { - regions.add(new HRegionInfo(table)); - } - servers.put(backupMaster, regions); - regions = new ArrayList<HRegionInfo>(); - for (int i=0; i<3; i++) { - regions.add(new HRegionInfo(table)); - } - servers.put(rs, regions); - plans = balancer.balanceCluster(servers); - assertNotNull(plans); - assertEquals(1, plans.size()); - } } http://git-wip-us.apache.org/repos/asf/hbase/blob/683f3b3d/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index e7757f5..4ff5afe 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -194,7 +194,7 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { public void testSkewCost() { Configuration conf = HBaseConfiguration.create(); StochasticLoadBalancer.CostFunction - costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf, 1); + costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf); for (int[] mockCluster : clusterStateMocks) { costFunction.init(mockCluster(mockCluster)); double cost = costFunction.cost(); @@ -319,7 +319,7 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { BaseLoadBalancer.Cluster cluster; - cluster = new BaseLoadBalancer.Cluster(master, clusterState, null, null, null, null, null); + cluster = new BaseLoadBalancer.Cluster(master, clusterState, null, null, null, null); costFunction.init(cluster); double costWithoutReplicas = costFunction.cost(); assertEquals(0, costWithoutReplicas, 0); @@ -329,7 +329,7 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { clusterState.firstEntry().getValue().get(0),1); clusterState.lastEntry().getValue().add(replica1); - cluster = new BaseLoadBalancer.Cluster(master, clusterState, null, null, null, null, null); + cluster = new BaseLoadBalancer.Cluster(master, clusterState, null, null, null, null); costFunction.init(cluster); double costWith1ReplicaDifferentServer = costFunction.cost(); @@ -339,7 +339,7 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { HRegionInfo replica2 = RegionReplicaUtil.getRegionInfoForReplica(replica1, 2); clusterState.lastEntry().getValue().add(replica2); - cluster = new BaseLoadBalancer.Cluster(master, clusterState, null, null, null, null, null); + cluster = new BaseLoadBalancer.Cluster(master, clusterState, null, null, null, null); costFunction.init(cluster); double costWith1ReplicaSameServer = costFunction.cost(); @@ -362,7 +362,7 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { entry.getValue().add(replica2); it.next().getValue().add(replica3); //2nd server - cluster = new BaseLoadBalancer.Cluster(master, clusterState, null, null, null, null, null); + cluster = new BaseLoadBalancer.Cluster(master, clusterState, null, null, null, null); costFunction.init(cluster); double costWith3ReplicasSameServer = costFunction.cost(); @@ -376,7 +376,7 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { clusterState.lastEntry().getValue().add(replica2); clusterState.lastEntry().getValue().add(replica3); - cluster = new BaseLoadBalancer.Cluster(master, clusterState, null, null, null, null, null); + cluster = new BaseLoadBalancer.Cluster(master, clusterState, null, null, null, null); costFunction.init(cluster); double costWith2ReplicasOnTwoServers = costFunction.cost(); @@ -396,7 +396,7 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { // until the step above s1 holds two replicas of a region regions = randomRegions(1); map.put(s2, regions); - assertTrue(loadBalancer.needsBalance(new Cluster(master, map, null, null, null, null, null))); + assertTrue(loadBalancer.needsBalance(new Cluster(master, map, null, null, null, null))); // check for the case where there are two hosts on the same rack and there are two racks // and both the replicas are on the same rack map.clear(); @@ -407,7 +407,7 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { map.put(s2, regionsOnS2); // add another server so that the cluster has some host on another rack map.put(ServerName.valueOf("host2", 1000, 11111), randomRegions(1)); - assertTrue(loadBalancer.needsBalance(new Cluster(master, map, null, null, null, null, + assertTrue(loadBalancer.needsBalance(new Cluster(master, map, null, null, null, new ForTestRackManagerOne()))); } http://git-wip-us.apache.org/repos/asf/hbase/blob/683f3b3d/src/main/docbkx/getting_started.xml ---------------------------------------------------------------------- diff --git a/src/main/docbkx/getting_started.xml b/src/main/docbkx/getting_started.xml index 2e0ee4e..79478ba 100644 --- a/src/main/docbkx/getting_started.xml +++ b/src/main/docbkx/getting_started.xml @@ -348,23 +348,14 @@ $ <title>Configure HBase.</title> <para> Edit the <filename>hbase-site.xml</filename> configuration. First, add the following - properties. Property <code>hbase.cluster.distributed</code> is set to <code>true</code> - (Its default is <code>false</code>), which directs HBase to run in distributed mode, - with one JVM instance per daemon. Since HBase version 1.0.0, a HMaster is also a - RegionServer. So in pseudo-distributed mode, just one HMaster (also a RegionServer) - instance is started by default. Because there is just one RegionServer (the HMaster), - property <code>hbase.master.wait.on.regionservers.mintostart</code> should be set to - <code>1</code> (Its default is changed to <code>2</code> since version 1.0.0). + property. which directs HBase to run in distributed mode, with one JVM instance per + daemon. </para> <programlisting language="xml"><![CDATA[ <property> <name>hbase.cluster.distributed</name> <value>true</value> </property> -<property> - <name>hbase.master.wait.on.regionservers.mintostart</name> - <value>1</value> -</property> ]]></programlisting> <para>Next, change the <code>hbase.rootdir</code> from the local filesystem to the address of your HDFS instance, using the <code>hdfs:////</code> URI syntax. In this example,
