Repository: hbase Updated Branches: refs/heads/branch-1 d3bccd9b9 -> d283818c5
HBASE-11604 Disable co-locating meta/master by default Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d283818c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d283818c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d283818c Branch: refs/heads/branch-1 Commit: d283818c51d8e69a77972c880369fdbd45b3297d Parents: d3bccd9 Author: Jimmy Xiang <[email protected]> Authored: Mon Jul 28 14:34:46 2014 -0700 Committer: Jimmy Xiang <[email protected]> Committed: Thu Sep 11 13:43:40 2014 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/master/ServerManager.java | 12 ++++- .../hbase/master/balancer/BaseLoadBalancer.java | 17 ++----- .../master/balancer/StochasticLoadBalancer.java | 1 + .../hadoop/hbase/client/TestScannerTimeout.java | 2 + .../hadoop/hbase/fs/TestBlockReorder.java | 2 +- .../master/TestAssignmentManagerOnCluster.java | 4 +- .../master/TestDistributedLogSplitting.java | 1 + .../hadoop/hbase/master/TestMasterFailover.java | 51 ++++++-------------- .../hadoop/hbase/master/TestMasterMetrics.java | 4 +- .../TestMasterOperationsForRegionReplicas.java | 4 +- .../hbase/master/TestZKLessAMOnCluster.java | 2 + .../master/balancer/TestBaseLoadBalancer.java | 26 +--------- .../TestRSKilledWhenInitializing.java | 2 +- .../regionserver/TestRegionServerMetrics.java | 5 ++ .../regionserver/TestRegionServerNoMaster.java | 19 -------- .../TestSplitTransactionOnCluster.java | 6 +-- ...isibilityLabelsWithDistributedLogReplay.java | 2 + src/main/docbkx/getting_started.xml | 13 +---- 18 files changed, 56 insertions(+), 117 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 6204206..ca14571 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -898,8 +898,18 @@ public class ServerManager { getLong(WAIT_ON_REGIONSERVERS_INTERVAL, 1500); final long timeout = this.master.getConfiguration(). getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500); + String[] tablesOnMaster = this.master.getConfiguration(). + getStrings("hbase.balancer.tablesOnMaster"); + int defaultMinToStart = 1; + if (tablesOnMaster != null && tablesOnMaster.length > 0) { + // If we assign regions to master, we'd like to start + // at least another region server so that we don't + // assign all regions to master if that region server + // doesn't come up in time. + defaultMinToStart = 2; + } int minToStart = this.master.getConfiguration(). - getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 2); + getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, defaultMinToStart); if (minToStart < 1) { LOG.warn(String.format( "The value of '%s' (%d) can not be less than 1, ignoring.", http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index d0c0a92..8af5c88 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -43,12 +43,10 @@ import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.RegionLoad; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; -import org.apache.hadoop.hbase.security.access.AccessControlLists; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action.Type; @@ -863,18 +861,12 @@ public abstract class BaseLoadBalancer implements LoadBalancer { // That's why the default activeMasterWeight is high. public static final String BACKUP_MASTER_WEIGHT_KEY = "hbase.balancer.backupMasterWeight"; - public static final int DEFAULT_BACKUP_MASTER_WEIGHT = 1; + public static final int DEFAULT_BACKUP_MASTER_WEIGHT = 0; private static final String ACTIVE_MASTER_WEIGHT_KEY = "hbase.balancer.activeMasterWeight"; private static final int DEFAULT_ACTIVE_MASTER_WEIGHT = 200; - // Regions of these tables are put on the master by default. - private static final String[] DEFAULT_TABLES_ON_MASTER = - new String[] {AccessControlLists.ACL_TABLE_NAME.getNameAsString(), - TableName.NAMESPACE_TABLE_NAME.getNameAsString(), - TableName.META_TABLE_NAME.getNameAsString()}; - protected int activeMasterWeight; protected int backupMasterWeight; @@ -906,8 +898,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { + BACKUP_MASTER_WEIGHT_KEY + " is " + backupMasterWeight + "(<1)"); } - String[] tables = conf.getStrings( - "hbase.balancer.tablesOnMaster", DEFAULT_TABLES_ON_MASTER); + String[] tables = conf.getStrings("hbase.balancer.tablesOnMaster"); if (tables != null) { for (String table: tables) { tablesOnMaster.add(table); @@ -1139,8 +1130,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer { int total = regions.size(); // Get the number of regions to be assigned // to backup masters based on the weight - int numRegions = total * numBackupMasters - / (numServers * backupMasterWeight + numBackupMasters); + int numRegions = usingBackupMasters ? total * numBackupMasters + / (numServers * backupMasterWeight + numBackupMasters) : 0; if (numRegions > 0) { // backupMasters can't be null, according to the formula, numBackupMasters != 0 roundRobinAssignment(cluster, regions, unassignedRegions, 0, http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 52db329..d32094d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -329,6 +329,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { loads = new HashMap<String, Deque<RegionLoad>>(); for (ServerName sn : clusterStatus.getServers()) { + if (excludedServers.contains(sn)) continue; ServerLoad sl = clusterStatus.getLoad(sn); if (sl == null) { continue; http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java index 402f798..e220058 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java @@ -63,6 +63,8 @@ public class TestScannerTimeout { Configuration c = TEST_UTIL.getConfiguration(); c.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, SCANNER_TIMEOUT); c.setInt(HConstants.THREAD_WAKE_FREQUENCY, THREAD_WAKE_FREQUENCY); + // Put meta on master to avoid meta server shutdown handling + c.set("hbase.balancer.tablesOnMaster", "hbase:meta"); // We need more than one region server for this test TEST_UTIL.startMiniCluster(2); Table table = TEST_UTIL.createTable(TABLE_NAME, SOME_BYTES); http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java index f83fbf3..b822a3d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java @@ -243,7 +243,7 @@ public class TestBlockReorder { byte[] sb = "sb".getBytes(); htu.startMiniZKCluster(); - MiniHBaseCluster hbm = htu.startMiniHBaseCluster(1, 0); + MiniHBaseCluster hbm = htu.startMiniHBaseCluster(1, 1); hbm.waitForActiveAndReadyMaster(); HRegionServer targetRs = hbm.getMaster(); http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java index 2208bf2..1c156a3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java @@ -70,7 +70,6 @@ import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; -import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; @@ -126,7 +125,8 @@ public class TestAssignmentManagerOnCluster { RegionStates regionStates = master.getAssignmentManager().getRegionStates(); ServerName metaServerName = regionStates.getRegionServerOfRegion( HRegionInfo.FIRST_META_REGIONINFO); - if (master.getServerName().equals(metaServerName)) { + if (master.getServerName().equals(metaServerName) || metaServerName == null + || !metaServerName.equals(cluster.getServerHoldingMeta())) { // Move meta off master metaServerName = cluster.getLiveRegionServerThreads() .get(0).getRegionServer().getServerName(); http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index b5400b2..5bdd4d1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -885,6 +885,7 @@ public class TestDistributedLogSplitting { List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); if (regions.isEmpty()) continue; region = regions.get(0); + if (region.isMetaRegion()) continue; regionSet.add(region); dstRS = rsts.get((i+1) % NUM_RS).getRegionServer(); break; http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 48e2f6f..6ca7d52 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -68,7 +68,6 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; @@ -293,22 +292,6 @@ public class TestMasterFailover { * javadoc. */ - // Master is down, so is the meta. We need to assign it somewhere - // so that regions can be assigned during the mocking phase. - ZKAssign.createNodeOffline( - zkw, HRegionInfo.FIRST_META_REGIONINFO, hrs.getServerName()); - ProtobufUtil.openRegion(hrs.getRSRpcServices(), - hrs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO); - - MetaTableLocator mtl = new MetaTableLocator(); - while (true) { - ServerName sn = mtl.getMetaRegionLocation(zkw); - if (sn != null && sn.equals(hrs.getServerName())) { - break; - } - Thread.sleep(100); - } - List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>(); List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>(); @@ -686,22 +669,6 @@ public class TestMasterFailover { * javadoc. */ - // Master is down, so is the meta. We need to assign it somewhere - // so that regions can be assigned during the mocking phase. - ZKAssign.createNodeOffline( - zkw, HRegionInfo.FIRST_META_REGIONINFO, hrs.getServerName()); - ProtobufUtil.openRegion(hrs.getRSRpcServices(), - hrs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO); - - MetaTableLocator mtl = new MetaTableLocator(); - while (true) { - ServerName sn = mtl.getMetaRegionLocation(zkw); - if (sn != null && sn.equals(hrs.getServerName())) { - break; - } - Thread.sleep(100); - } - List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>(); List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>(); @@ -988,16 +955,26 @@ public class TestMasterFailover { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); // Find regionserver carrying meta. - HRegionServer regionServer = cluster.getMaster(); - HRegion metaRegion = regionServer.getOnlineRegion( - HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); + List<RegionServerThread> regionServerThreads = + cluster.getRegionServerThreads(); + HRegion metaRegion = null; + HRegionServer metaRegionServer = null; + for (RegionServerThread regionServerThread : regionServerThreads) { + HRegionServer regionServer = regionServerThread.getRegionServer(); + metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); + regionServer.abort(""); + if (null != metaRegion) { + metaRegionServer = regionServer; + break; + } + } TEST_UTIL.shutdownMiniHBaseCluster(); // Create a ZKW to use in the test ZooKeeperWatcher zkw = HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL, - metaRegion, regionServer.getServerName()); + metaRegion, metaRegionServer.getServerName()); LOG.info("Staring cluster for second time"); TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS); http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index 2bae847..2ee7d51 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -66,7 +66,7 @@ public class TestMasterMetrics { public static void startCluster() throws Exception { LOG.info("Starting cluster"); TEST_UTIL = new HBaseTestingUtility(); - TEST_UTIL.startMiniCluster(1, 0, 1, null, MyMaster.class, null); + TEST_UTIL.startMiniCluster(1, 1, 1, null, MyMaster.class, null); cluster = TEST_UTIL.getHBaseCluster(); LOG.info("Waiting for active/ready master"); cluster.waitForActiveAndReadyMaster(); @@ -116,7 +116,7 @@ public class TestMasterMetrics { @Test public void testDefaultMasterMetrics() throws Exception { MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource(); - metricsHelper.assertGauge( "numRegionServers", 1, masterSource); + metricsHelper.assertGauge( "numRegionServers", 2, masterSource); metricsHelper.assertGauge( "averageLoad", 2, masterSource); metricsHelper.assertGauge( "numDeadRegionServers", 0, masterSource); http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java index 5f409fe..34cc199 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java @@ -337,12 +337,12 @@ public class TestMasterOperationsForRegionReplicas { Map<HRegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap(); assertEquals(regionToServerMap.size(), numRegions * numReplica + 1); //'1' for the namespace Map<ServerName, List<HRegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap(); - assertEquals(serverToRegionMap.keySet().size(), 2); // 1 rs + 1 master + assertEquals(serverToRegionMap.keySet().size(), 1); // master by default not used for (Map.Entry<ServerName, List<HRegionInfo>> entry : serverToRegionMap.entrySet()) { if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) { continue; } - assertEquals(entry.getValue().size(), numRegions * numReplica); + assertEquals(entry.getValue().size(), numRegions * numReplica + 1); //'1' for the namespace } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestZKLessAMOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestZKLessAMOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestZKLessAMOnCluster.java index 83d33c5..6eb273a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestZKLessAMOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestZKLessAMOnCluster.java @@ -30,6 +30,8 @@ public class TestZKLessAMOnCluster extends TestAssignmentManagerOnCluster { @BeforeClass public static void setUpBeforeClass() throws Exception { + // Put meta on master to avoid meta server shutdown handling + conf.set("hbase.balancer.tablesOnMaster", "hbase:meta"); // Don't use ZK for region assignment conf.setBoolean("hbase.assignment.usezk", false); setupOnce(); http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java index 0507e26..625486d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.master.balancer; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -118,17 +117,6 @@ public class TestBaseLoadBalancer extends BalancerTestBase { */ @Test public void testImmediateAssignment() throws Exception { - List<ServerName> tmp = getListOfServerNames(randomServers(1, 0)); - tmp.add(master); - ServerName sn = loadBalancer.randomAssignment(HRegionInfo.FIRST_META_REGIONINFO, tmp); - assertEquals(master, sn); - HRegionInfo hri = randomRegions(1, -1).get(0); - sn = loadBalancer.randomAssignment(hri, tmp); - assertNotEquals(master, sn); - tmp = new ArrayList<ServerName>(); - tmp.add(master); - sn = loadBalancer.randomAssignment(hri, tmp); - assertEquals(master, sn); for (int[] mock : regionsAndServersMocks) { LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers"); List<HRegionInfo> regions = randomRegions(mock[0]); @@ -164,18 +152,6 @@ public class TestBaseLoadBalancer extends BalancerTestBase { */ @Test public void testBulkAssignment() throws Exception { - List<ServerName> tmp = getListOfServerNames(randomServers(5, 0)); - List<HRegionInfo> hris = randomRegions(20); - hris.add(HRegionInfo.FIRST_META_REGIONINFO); - tmp.add(master); - Map<ServerName, List<HRegionInfo>> plans = loadBalancer.roundRobinAssignment(hris, tmp); - assertTrue(plans.get(master).contains(HRegionInfo.FIRST_META_REGIONINFO)); - assertEquals(1, plans.get(master).size()); - int totalRegion = 0; - for (List<HRegionInfo> regions: plans.values()) { - totalRegion += regions.size(); - } - assertEquals(hris.size(), totalRegion); for (int[] mock : regionsAndServersMocks) { LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers"); List<HRegionInfo> regions = randomRegions(mock[0]); @@ -533,7 +509,7 @@ public class TestBaseLoadBalancer extends BalancerTestBase { balancer.usingBackupMasters = false; Mockito.when(st.getBackupMasters()).thenReturn(backupMasters); loadBalancer.setClusterStatus(st); - assertEquals(1, balancer.excludedServers.size()); + assertEquals(backupMasters.size(), balancer.excludedServers.size()); assertTrue(balancer.excludedServers.contains(backupMaster)); // Round robin assignment http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java index 16c8681..87311a0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java @@ -76,7 +76,7 @@ public class TestRSKilledWhenInitializing { master.start(); try { long startTime = System.currentTimeMillis(); - while (!master.getMaster().isInitialized()) { + while (!master.getMaster().isActiveMaster()) { try { Thread.sleep(100); } catch (InterruptedException ignored) { http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index 7ae0b5b..ee5c4db 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -158,6 +158,11 @@ public class TestRegionServerMetrics { } table.get(gets); + // By default, master doesn't host meta now. + // Adding some meta related requests + requests += 3; + readRequests ++; + metricsRegionServer.getRegionServerWrapper().forceRecompute(); metricsHelper.assertCounter("totalRequestCount", requests + 50, serverSource); metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource); http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java index 0843abe..4291590 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager; import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination; import org.apache.hadoop.hbase.executor.EventType; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.RequestConverter; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest; @@ -43,9 +42,7 @@ import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKAssign; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.junit.After; @@ -103,22 +100,6 @@ public class TestRegionServerNoMaster { && HTU.getHBaseCluster().getMasterThread().isAlive()) { Threads.sleep(100); } - // Master is down, so is the meta. We need to assign it somewhere - // so that regions can be assigned during the mocking phase. - HRegionServer hrs = HTU.getHBaseCluster().getRegionServer(0); - ZooKeeperWatcher zkw = hrs.getZooKeeper(); - ZKAssign.createNodeOffline( - zkw, HRegionInfo.FIRST_META_REGIONINFO, hrs.getServerName()); - ProtobufUtil.openRegion(hrs.getRSRpcServices(), - hrs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO); - MetaTableLocator mtl = new MetaTableLocator(); - while (true) { - ServerName sn = mtl.getMetaRegionLocation(zkw); - if (sn != null && sn.equals(hrs.getServerName())) { - break; - } - Thread.sleep(100); - } } /** Flush the given region in the mini cluster. Since no master, we cannot use HBaseAdmin.flush() */ http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index edbdd64..20cc8e5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -131,7 +131,7 @@ public class TestSplitTransactionOnCluster { static void setupOnce() throws Exception { TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000); useZKForAssignment = TESTING_UTIL.getConfiguration().getBoolean( - "hbase.assignment.usezk", false); + "hbase.assignment.usezk", true); TESTING_UTIL.startMiniCluster(NB_SERVERS); } @@ -1270,8 +1270,8 @@ public class TestSplitTransactionOnCluster { // hbase:meta We don't want hbase:meta replay polluting our test when we later crash // the table region serving server. int metaServerIndex = cluster.getServerWithMeta(); - assertTrue(metaServerIndex == -1); // meta is on master now - HRegionServer metaRegionServer = cluster.getMaster(); + assertTrue(metaServerIndex != -1); + HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex); int tableRegionIndex = cluster.getServerWith(hri.getRegionName()); assertTrue(tableRegionIndex != -1); HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex); http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java index 11a830c..3696a04 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java @@ -42,6 +42,8 @@ public class TestVisibilityLabelsWithDistributedLogReplay extends conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, ScanLabelGenerator.class); conf.set("hbase.superuser", "admin"); + // Put meta on master to avoid meta server shutdown handling + conf.set("hbase.balancer.tablesOnMaster", "hbase:meta"); TEST_UTIL.startMiniCluster(2); SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); USER1 = User.createUserForTesting(conf, "user1", new String[] {}); http://git-wip-us.apache.org/repos/asf/hbase/blob/d283818c/src/main/docbkx/getting_started.xml ---------------------------------------------------------------------- diff --git a/src/main/docbkx/getting_started.xml b/src/main/docbkx/getting_started.xml index b4e9911..9ea9f11 100644 --- a/src/main/docbkx/getting_started.xml +++ b/src/main/docbkx/getting_started.xml @@ -327,23 +327,14 @@ $ <title>Configure HBase.</title> <para> Edit the <filename>hbase-site.xml</filename> configuration. First, add the following - properties. Property <code>hbase.cluster.distributed</code> is set to <code>true</code> - (Its default is <code>false</code>), which directs HBase to run in distributed mode, - with one JVM instance per daemon. Since HBase version 1.0.0, a HMaster is also a - RegionServer. So in pseudo-distributed mode, just one HMaster (also a RegionServer) - instance is started by default. Because there is just one RegionServer (the HMaster), - property <code>hbase.master.wait.on.regionservers.mintostart</code> should be set to - <code>1</code> (Its default is changed to <code>2</code> since version 1.0.0). + property, which directs HBase to run in distributed mode, with one JVM instance per + daemon. </para> <programlisting><![CDATA[ <property> <name>hbase.cluster.distributed</name> <value>true</value> </property> -<property> - <name>hbase.master.wait.on.regionservers.mintostart</name> - <value>1</value> -</property> ]]></programlisting> <para>Next, change the <code>hbase.rootdir</code> from the local filesystem to the address of your HDFS instance, using the <code>hdfs:////</code> URI syntax. In this example,
